diff options
Diffstat (limited to 'python/mach/mach/test')
25 files changed, 1875 insertions, 0 deletions
diff --git a/python/mach/mach/test/__init__.py b/python/mach/mach/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/test/__init__.py diff --git a/python/mach/mach/test/conftest.py b/python/mach/mach/test/conftest.py new file mode 100644 index 0000000000..78129acb58 --- /dev/null +++ b/python/mach/mach/test/conftest.py @@ -0,0 +1,84 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import unittest +from collections.abc import Iterable +from pathlib import Path +from typing import List, Optional, Union + +import pytest +from buildconfig import topsrcdir + +try: + from StringIO import StringIO +except ImportError: + # TODO io.StringIO causes failures with Python 2 (needs to be sorted out) + from io import StringIO + +from mach.main import Mach + +PROVIDER_DIR = Path(__file__).resolve().parent / "providers" + + +@pytest.fixture(scope="class") +def get_mach(request): + def _populate_context(key): + if key == "topdir": + return topsrcdir + + def inner( + provider_files: Optional[Union[Path, List[Path]]] = None, + entry_point=None, + context_handler=None, + ): + m = Mach(str(Path.cwd())) + m.define_category("testing", "Mach unittest", "Testing for mach core", 10) + m.define_category("misc", "Mach misc", "Testing for mach core", 20) + m.populate_context_handler = context_handler or _populate_context + + if provider_files: + if not isinstance(provider_files, Iterable): + provider_files = [provider_files] + + for path in provider_files: + m.load_commands_from_file(PROVIDER_DIR / path) + + if entry_point: + m.load_commands_from_entry_point(entry_point) + + return m + + if request.cls and issubclass(request.cls, unittest.TestCase): + request.cls.get_mach = lambda cls, *args, **kwargs: inner(*args, **kwargs) + return inner + + +@pytest.fixture(scope="class") +def run_mach(request, get_mach): + def inner(argv, *args, **kwargs): + m = get_mach(*args, **kwargs) + + stdout = StringIO() + stderr = StringIO() + + if sys.version_info < (3, 0): + stdout.encoding = "UTF-8" + stderr.encoding = "UTF-8" + + try: + result = m.run(argv, stdout=stdout, stderr=stderr) + except SystemExit: + result = None + + return (result, stdout.getvalue(), stderr.getvalue()) + + if request.cls and issubclass(request.cls, unittest.TestCase): + request.cls._run_mach = lambda cls, *args, **kwargs: inner(*args, **kwargs) + return inner + + +@pytest.mark.usefixtures("get_mach", "run_mach") +class TestBase(unittest.TestCase): + pass diff --git a/python/mach/mach/test/invoke_mach_command.py b/python/mach/mach/test/invoke_mach_command.py new file mode 100644 index 0000000000..1efa102ef5 --- /dev/null +++ b/python/mach/mach/test/invoke_mach_command.py @@ -0,0 +1,4 @@ +import subprocess +import sys + +subprocess.check_call([sys.executable] + sys.argv[1:]) diff --git a/python/mach/mach/test/providers/__init__.py b/python/mach/mach/test/providers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/python/mach/mach/test/providers/__init__.py diff --git a/python/mach/mach/test/providers/basic.py b/python/mach/mach/test/providers/basic.py new file mode 100644 index 0000000000..26cdfdf588 --- /dev/null +++ b/python/mach/mach/test/providers/basic.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mach.decorators import Command, CommandArgument + + +@Command("cmd_foo", category="testing") +def run_foo(command_context): + pass + + +@Command("cmd_bar", category="testing") +@CommandArgument("--baz", action="store_true", help="Run with baz") +def run_bar(command_context, baz=None): + pass diff --git a/python/mach/mach/test/providers/commands.py b/python/mach/mach/test/providers/commands.py new file mode 100644 index 0000000000..6b8210c513 --- /dev/null +++ b/python/mach/mach/test/providers/commands.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from functools import partial + +from mach.decorators import Command, CommandArgument + + +def is_foo(cls): + """Foo must be true""" + return cls.foo + + +def is_bar(val, cls): + """Bar must equal val""" + return cls.bar == val + + +@Command("cmd_foo", category="testing") +@CommandArgument("--arg", default=None, help="Argument help.") +def run_foo(command_context): + pass + + +@Command("cmd_bar", category="testing", conditions=[partial(is_bar, False)]) +def run_bar(command_context): + pass + + +@Command("cmd_foobar", category="testing", conditions=[is_foo, partial(is_bar, True)]) +def run_foobar(command_context): + pass diff --git a/python/mach/mach/test/providers/conditions.py b/python/mach/mach/test/providers/conditions.py new file mode 100644 index 0000000000..db2f3f8123 --- /dev/null +++ b/python/mach/mach/test/providers/conditions.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mach.decorators import Command + + +def is_true(cls): + return True + + +def is_false(cls): + return False + + +@Command("cmd_condition_true", category="testing", conditions=[is_true]) +def run_condition_true(self, command_context): + pass + + +@Command("cmd_condition_false", category="testing", conditions=[is_false]) +def run_condition_false(self, command_context): + pass + + +@Command( + "cmd_condition_true_and_false", category="testing", conditions=[is_true, is_false] +) +def run_condition_true_and_false(self, command_context): + pass + + +def is_ctx_foo(cls): + """Foo must be true""" + return cls._mach_context.foo + + +def is_ctx_bar(cls): + """Bar must be true""" + return cls._mach_context.bar + + +@Command("cmd_foo_ctx", category="testing", conditions=[is_ctx_foo]) +def run_foo_ctx(self, command_context): + pass + + +@Command("cmd_bar_ctx", category="testing", conditions=[is_ctx_bar]) +def run_bar_ctx(self, command_context): + pass + + +@Command("cmd_foobar_ctx", category="testing", conditions=[is_ctx_foo, is_ctx_bar]) +def run_foobar_ctx(self, command_context): + pass diff --git a/python/mach/mach/test/providers/conditions_invalid.py b/python/mach/mach/test/providers/conditions_invalid.py new file mode 100644 index 0000000000..228c56f0bf --- /dev/null +++ b/python/mach/mach/test/providers/conditions_invalid.py @@ -0,0 +1,10 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mach.decorators import Command + + +@Command("cmd_foo", category="testing", conditions=["invalid"]) +def run_foo(command_context): + pass diff --git a/python/mach/mach/test/providers/throw.py b/python/mach/mach/test/providers/throw.py new file mode 100644 index 0000000000..9ddc7653c0 --- /dev/null +++ b/python/mach/mach/test/providers/throw.py @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mach.decorators import Command, CommandArgument +from mach.test.providers import throw2 + + +@Command("throw", category="testing") +@CommandArgument("--message", "-m", default="General Error") +def throw(command_context, message): + raise Exception(message) + + +@Command("throw_deep", category="testing") +@CommandArgument("--message", "-m", default="General Error") +def throw_deep(command_context, message): + throw2.throw_deep(message) diff --git a/python/mach/mach/test/providers/throw2.py b/python/mach/mach/test/providers/throw2.py new file mode 100644 index 0000000000..9ff7f2798e --- /dev/null +++ b/python/mach/mach/test/providers/throw2.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file exists to trigger the differences in mach error reporting between +# exceptions that occur in mach command modules themselves and in the things +# they call. + + +def throw_deep(message): + return throw_real(message) + + +def throw_real(message): + raise Exception(message) diff --git a/python/mach/mach/test/python.ini b/python/mach/mach/test/python.ini new file mode 100644 index 0000000000..de09924b67 --- /dev/null +++ b/python/mach/mach/test/python.ini @@ -0,0 +1,22 @@ +[DEFAULT] +subsuite = mach + +[test_commands.py] +[test_conditions.py] +skip-if = python == 3 +[test_config.py] +[test_decorators.py] +[test_dispatcher.py] +[test_entry_point.py] +[test_error_output.py] +skip-if = python == 3 +[test_logger.py] +[test_mach.py] +[test_site.py] +[test_site_activation.py] +[test_site_compatibility.py] +# The Windows and Mac workers only use the internal PyPI mirror, +# which will be missing packages required for this test. +skip-if = + os == "win" + os == "mac" diff --git a/python/mach/mach/test/script_site_activation.py b/python/mach/mach/test/script_site_activation.py new file mode 100644 index 0000000000..8c23f1a19c --- /dev/null +++ b/python/mach/mach/test/script_site_activation.py @@ -0,0 +1,67 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script is used by "test_site_activation.py" to verify how site activations +# affect the sys.path. +# The sys.path is printed in three stages: +# 1. Once at the beginning +# 2. Once after Mach site activation +# 3. Once after the command site activation +# The output of this script should be an ast-parsable list with three nested lists: one +# for each sys.path state. +# Note that virtualenv-creation output may need to be filtered out - it can be done by +# only ast-parsing the last line of text outputted by this script. + +import os +import sys +from unittest.mock import patch + +from mach.requirements import MachEnvRequirements, PthSpecifier +from mach.site import CommandSiteManager, MachSiteManager + + +def main(): + # Should be set by calling test + topsrcdir = os.environ["TOPSRCDIR"] + command_site = os.environ["COMMAND_SITE"] + mach_site_requirements = os.environ["MACH_SITE_PTH_REQUIREMENTS"] + command_site_requirements = os.environ["COMMAND_SITE_PTH_REQUIREMENTS"] + work_dir = os.environ["WORK_DIR"] + + def resolve_requirements(topsrcdir, site_name): + req = MachEnvRequirements() + if site_name == "mach": + req.pth_requirements = [ + PthSpecifier(path) for path in mach_site_requirements.split(os.pathsep) + ] + else: + req.pth_requirements = [PthSpecifier(command_site_requirements)] + return req + + with patch("mach.site.resolve_requirements", resolve_requirements): + initial_sys_path = sys.path.copy() + + mach_site = MachSiteManager.from_environment( + topsrcdir, + lambda: work_dir, + ) + mach_site.activate() + mach_sys_path = sys.path.copy() + + command_site = CommandSiteManager.from_environment( + topsrcdir, lambda: work_dir, command_site, work_dir + ) + command_site.activate() + command_sys_path = sys.path.copy() + print( + [ + initial_sys_path, + mach_sys_path, + command_sys_path, + ] + ) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_commands.py b/python/mach/mach/test/test_commands.py new file mode 100644 index 0000000000..38191b0898 --- /dev/null +++ b/python/mach/mach/test/test_commands.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +from pathlib import Path + +import pytest +from buildconfig import topsrcdir +from mozunit import main + +import mach + +ALL_COMMANDS = [ + "cmd_bar", + "cmd_foo", + "cmd_foobar", + "mach-commands", + "mach-completion", + "mach-debug-commands", +] + + +@pytest.fixture +def run_completion(run_mach): + def inner(args=[]): + mach_dir = Path(mach.__file__).parent + providers = [ + Path("commands.py"), + mach_dir / "commands" / "commandinfo.py", + ] + + def context_handler(key): + if key == "topdir": + return topsrcdir + + args = ["mach-completion"] + args + return run_mach(args, providers, context_handler=context_handler) + + return inner + + +def format(targets): + return "\n".join(targets) + "\n" + + +def test_mach_completion(run_completion): + result, stdout, stderr = run_completion() + assert result == 0 + assert stdout == format(ALL_COMMANDS) + + result, stdout, stderr = run_completion(["cmd_f"]) + assert result == 0 + # While it seems like this should return only commands that have + # 'cmd_f' as a prefix, the completion script will handle this case + # properly. + assert stdout == format(ALL_COMMANDS) + + result, stdout, stderr = run_completion(["cmd_foo"]) + assert result == 0 + assert stdout == format(["help", "--arg"]) + + +@pytest.mark.parametrize("shell", ("bash", "fish", "zsh")) +def test_generate_mach_completion_script(run_completion, shell): + rv, out, err = run_completion([shell]) + print(out) + print(err, file=sys.stderr) + assert rv == 0 + assert err == "" + + assert "cmd_foo" in out + assert "arg" in out + assert "cmd_foobar" in out + assert "cmd_bar" in out + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_conditions.py b/python/mach/mach/test/test_conditions.py new file mode 100644 index 0000000000..5775790e69 --- /dev/null +++ b/python/mach/mach/test/test_conditions.py @@ -0,0 +1,101 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path + +from buildconfig import topsrcdir +from mozunit import main + +from mach.base import MachError +from mach.main import Mach +from mach.registrar import Registrar +from mach.test.conftest import PROVIDER_DIR, TestBase + + +def _make_populate_context(include_extra_attributes): + def _populate_context(key=None): + if key is None: + return + + attributes = { + "topdir": topsrcdir, + } + if include_extra_attributes: + attributes["foo"] = True + attributes["bar"] = False + + try: + return attributes[key] + except KeyError: + raise AttributeError(key) + + return _populate_context + + +_populate_bare_context = _make_populate_context(False) +_populate_context = _make_populate_context(True) + + +class TestConditions(TestBase): + """Tests for conditionally filtering commands.""" + + def _run(self, args, context_handler=_populate_bare_context): + return self._run_mach( + args, Path("conditions.py"), context_handler=context_handler + ) + + def test_conditions_pass(self): + """Test that a command which passes its conditions is runnable.""" + + self.assertEqual((0, "", ""), self._run(["cmd_condition_true"])) + self.assertEqual((0, "", ""), self._run(["cmd_foo_ctx"], _populate_context)) + + def test_invalid_context_message(self): + """Test that commands which do not pass all their conditions + print the proper failure message.""" + + def is_bar(): + """Bar must be true""" + + fail_conditions = [is_bar] + + for name in ("cmd_condition_false", "cmd_condition_true_and_false"): + result, stdout, stderr = self._run([name]) + self.assertEqual(1, result) + + fail_msg = Registrar._condition_failed_message(name, fail_conditions) + self.assertEqual(fail_msg.rstrip(), stdout.rstrip()) + + for name in ("cmd_bar_ctx", "cmd_foobar_ctx"): + result, stdout, stderr = self._run([name], _populate_context) + self.assertEqual(1, result) + + fail_msg = Registrar._condition_failed_message(name, fail_conditions) + self.assertEqual(fail_msg.rstrip(), stdout.rstrip()) + + def test_invalid_type(self): + """Test that a condition which is not callable raises an exception.""" + + m = Mach(str(Path.cwd())) + m.define_category("testing", "Mach unittest", "Testing for mach core", 10) + self.assertRaises( + MachError, + m.load_commands_from_file, + PROVIDER_DIR / "conditions_invalid.py", + ) + + def test_help_message(self): + """Test that commands that are not runnable do not show up in help.""" + + result, stdout, stderr = self._run(["help"], _populate_context) + self.assertIn("cmd_condition_true", stdout) + self.assertNotIn("cmd_condition_false", stdout) + self.assertNotIn("cmd_condition_true_and_false", stdout) + self.assertIn("cmd_foo_ctx", stdout) + self.assertNotIn("cmd_bar_ctx", stdout) + self.assertNotIn("cmd_foobar_ctx", stdout) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_config.py b/python/mach/mach/test/test_config.py new file mode 100644 index 0000000000..25b75c8685 --- /dev/null +++ b/python/mach/mach/test/test_config.py @@ -0,0 +1,292 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +import sys +import unittest +from pathlib import Path + +from mozfile.mozfile import NamedTemporaryFile +from mozunit import main +from six import string_types + +from mach.config import ( + BooleanType, + ConfigException, + ConfigSettings, + IntegerType, + PathType, + PositiveIntegerType, + StringType, +) +from mach.decorators import SettingsProvider + +CONFIG1 = r""" +[foo] + +bar = bar_value +baz = /baz/foo.c +""" + +CONFIG2 = r""" +[foo] + +bar = value2 +""" + + +@SettingsProvider +class Provider1(object): + config_settings = [ + ("foo.bar", StringType, "desc"), + ("foo.baz", PathType, "desc"), + ] + + +@SettingsProvider +class ProviderDuplicate(object): + config_settings = [ + ("dupesect.foo", StringType, "desc"), + ("dupesect.foo", StringType, "desc"), + ] + + +@SettingsProvider +class Provider2(object): + config_settings = [ + ("a.string", StringType, "desc"), + ("a.boolean", BooleanType, "desc"), + ("a.pos_int", PositiveIntegerType, "desc"), + ("a.int", IntegerType, "desc"), + ("a.path", PathType, "desc"), + ] + + +@SettingsProvider +class Provider3(object): + @classmethod + def config_settings(cls): + return [ + ("a.string", "string", "desc"), + ("a.boolean", "boolean", "desc"), + ("a.pos_int", "pos_int", "desc"), + ("a.int", "int", "desc"), + ("a.path", "path", "desc"), + ] + + +@SettingsProvider +class Provider4(object): + config_settings = [ + ("foo.abc", StringType, "desc", "a", {"choices": set("abc")}), + ("foo.xyz", StringType, "desc", "w", {"choices": set("xyz")}), + ] + + +@SettingsProvider +class Provider5(object): + config_settings = [ + ("foo.*", "string", "desc"), + ("foo.bar", "string", "desc"), + ] + + +class TestConfigSettings(unittest.TestCase): + def test_empty(self): + s = ConfigSettings() + + self.assertEqual(len(s), 0) + self.assertNotIn("foo", s) + + def test_duplicate_option(self): + s = ConfigSettings() + + with self.assertRaises(ConfigException): + s.register_provider(ProviderDuplicate) + + def test_simple(self): + s = ConfigSettings() + s.register_provider(Provider1) + + self.assertEqual(len(s), 1) + self.assertIn("foo", s) + + foo = s["foo"] + foo = s.foo + + self.assertEqual(len(foo), 0) + self.assertEqual(len(foo._settings), 2) + + self.assertIn("bar", foo._settings) + self.assertIn("baz", foo._settings) + + self.assertNotIn("bar", foo) + foo["bar"] = "value1" + self.assertIn("bar", foo) + + self.assertEqual(foo["bar"], "value1") + self.assertEqual(foo.bar, "value1") + + def test_assignment_validation(self): + s = ConfigSettings() + s.register_provider(Provider2) + + a = s.a + + # Assigning an undeclared setting raises. + exc_type = AttributeError if sys.version_info < (3, 0) else KeyError + with self.assertRaises(exc_type): + a.undefined = True + + with self.assertRaises(KeyError): + a["undefined"] = True + + # Basic type validation. + a.string = "foo" + a.string = "foo" + + with self.assertRaises(TypeError): + a.string = False + + a.boolean = True + a.boolean = False + + with self.assertRaises(TypeError): + a.boolean = "foo" + + a.pos_int = 5 + a.pos_int = 0 + + with self.assertRaises(ValueError): + a.pos_int = -1 + + with self.assertRaises(TypeError): + a.pos_int = "foo" + + a.int = 5 + a.int = 0 + a.int = -5 + + with self.assertRaises(TypeError): + a.int = 1.24 + + with self.assertRaises(TypeError): + a.int = "foo" + + a.path = "/home/gps" + a.path = "foo.c" + a.path = "foo/bar" + a.path = "./foo" + + def retrieval_type_helper(self, provider): + s = ConfigSettings() + s.register_provider(provider) + + a = s.a + + a.string = "foo" + a.boolean = True + a.pos_int = 12 + a.int = -4 + a.path = "./foo/bar" + + self.assertIsInstance(a.string, string_types) + self.assertIsInstance(a.boolean, bool) + self.assertIsInstance(a.pos_int, int) + self.assertIsInstance(a.int, int) + self.assertIsInstance(a.path, string_types) + + def test_retrieval_type(self): + self.retrieval_type_helper(Provider2) + self.retrieval_type_helper(Provider3) + + def test_choices_validation(self): + s = ConfigSettings() + s.register_provider(Provider4) + + foo = s.foo + foo.abc + with self.assertRaises(ValueError): + foo.xyz + + with self.assertRaises(ValueError): + foo.abc = "e" + + foo.abc = "b" + foo.xyz = "y" + + def test_wildcard_options(self): + s = ConfigSettings() + s.register_provider(Provider5) + + foo = s.foo + + self.assertIn("*", foo._settings) + self.assertNotIn("*", foo) + + foo.baz = "value1" + foo.bar = "value2" + + self.assertIn("baz", foo) + self.assertEqual(foo.baz, "value1") + + self.assertIn("bar", foo) + self.assertEqual(foo.bar, "value2") + + def test_file_reading_single(self): + temp = NamedTemporaryFile(mode="wt") + temp.write(CONFIG1) + temp.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_file(Path(temp.name)) + + self.assertEqual(s.foo.bar, "bar_value") + + def test_file_reading_multiple(self): + """Loading multiple files has proper overwrite behavior.""" + temp1 = NamedTemporaryFile(mode="wt") + temp1.write(CONFIG1) + temp1.flush() + + temp2 = NamedTemporaryFile(mode="wt") + temp2.write(CONFIG2) + temp2.flush() + + s = ConfigSettings() + s.register_provider(Provider1) + + s.load_files([Path(temp1.name), Path(temp2.name)]) + + self.assertEqual(s.foo.bar, "value2") + + def test_file_reading_missing(self): + """Missing files should silently be ignored.""" + + s = ConfigSettings() + + s.load_file("/tmp/foo.ini") + + def test_file_writing(self): + s = ConfigSettings() + s.register_provider(Provider2) + + s.a.string = "foo" + s.a.boolean = False + + temp = NamedTemporaryFile("wt") + s.write(temp) + temp.flush() + + s2 = ConfigSettings() + s2.register_provider(Provider2) + + s2.load_file(temp.name) + + self.assertEqual(s.a.string, s2.a.string) + self.assertEqual(s.a.boolean, s2.a.boolean) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_decorators.py b/python/mach/mach/test/test_decorators.py new file mode 100644 index 0000000000..f33b6e7d8f --- /dev/null +++ b/python/mach/mach/test/test_decorators.py @@ -0,0 +1,133 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path +from unittest import mock +from unittest.mock import Mock, patch + +import pytest +from mozbuild.base import MachCommandBase +from mozunit import main + +import mach.decorators +import mach.registrar +from mach.base import MachError +from mach.decorators import Command, CommandArgument, SubCommand +from mach.requirements import MachEnvRequirements +from mach.site import CommandSiteManager, MozSiteMetadata, SitePackagesSource + + +@pytest.fixture +def registrar(monkeypatch): + test_registrar = mach.registrar.MachRegistrar() + test_registrar.register_category( + "testing", "Mach unittest", "Testing for mach decorators" + ) + monkeypatch.setattr(mach.decorators, "Registrar", test_registrar) + return test_registrar + + +def test_register_command_with_argument(registrar): + inner_function = Mock() + context = Mock() + context.cwd = "." + + @Command("cmd_foo", category="testing") + @CommandArgument("--arg", default=None, help="Argument help.") + def run_foo(command_context, arg): + inner_function(arg) + + registrar.dispatch("cmd_foo", context, arg="argument") + + inner_function.assert_called_with("argument") + + +def test_register_command_with_metrics_path(registrar): + context = Mock() + context.cwd = "." + + metrics_path = "metrics/path" + metrics_mock = Mock() + context.telemetry.metrics.return_value = metrics_mock + + @Command("cmd_foo", category="testing", metrics_path=metrics_path) + def run_foo(command_context): + assert command_context.metrics == metrics_mock + + @SubCommand("cmd_foo", "sub_foo", metrics_path=metrics_path + "2") + def run_subfoo(command_context): + assert command_context.metrics == metrics_mock + + registrar.dispatch("cmd_foo", context) + + context.telemetry.metrics.assert_called_with(metrics_path) + assert context.handler.metrics_path == metrics_path + + registrar.dispatch("cmd_foo", context, subcommand="sub_foo") + assert context.handler.metrics_path == metrics_path + "2" + + +def test_register_command_sets_up_class_at_runtime(registrar): + inner_function = Mock() + + context = Mock() + context.cwd = "." + + # We test that the virtualenv is set up properly dynamically on + # the instance that actually runs the command. + @Command("cmd_foo", category="testing", virtualenv_name="env_foo") + def run_foo(command_context): + assert ( + Path(command_context.virtualenv_manager.virtualenv_root).name == "env_foo" + ) + inner_function("foo") + + @Command("cmd_bar", category="testing", virtualenv_name="env_bar") + def run_bar(command_context): + assert ( + Path(command_context.virtualenv_manager.virtualenv_root).name == "env_bar" + ) + inner_function("bar") + + def from_environment_patch( + topsrcdir: str, state_dir: str, virtualenv_name, directory: str + ): + return CommandSiteManager( + "", + "", + virtualenv_name, + virtualenv_name, + MozSiteMetadata(0, "mach", SitePackagesSource.VENV, "", ""), + True, + MachEnvRequirements(), + ) + + with mock.patch.object( + CommandSiteManager, "from_environment", from_environment_patch + ): + with patch.object(MachCommandBase, "activate_virtualenv"): + registrar.dispatch("cmd_foo", context) + inner_function.assert_called_with("foo") + registrar.dispatch("cmd_bar", context) + inner_function.assert_called_with("bar") + + +def test_cannot_create_command_nonexisting_category(registrar): + with pytest.raises(MachError): + + @Command("cmd_foo", category="bar") + def run_foo(command_context): + pass + + +def test_subcommand_requires_parent_to_exist(registrar): + with pytest.raises(MachError): + + @SubCommand("sub_foo", "foo") + def run_foo(command_context): + pass + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_dispatcher.py b/python/mach/mach/test/test_dispatcher.py new file mode 100644 index 0000000000..85c2e9a847 --- /dev/null +++ b/python/mach/mach/test/test_dispatcher.py @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from io import StringIO +from pathlib import Path + +import pytest +from mozunit import main +from six import string_types + +from mach.base import CommandContext +from mach.registrar import Registrar + + +@pytest.mark.usefixtures("get_mach", "run_mach") +class TestDispatcher(unittest.TestCase): + """Tests dispatch related code""" + + def get_parser(self, config=None): + mach = self.get_mach(Path("basic.py")) + + for provider in Registrar.settings_providers: + mach.settings.register_provider(provider) + + if config: + if isinstance(config, string_types): + config = StringIO(config) + mach.settings.load_fps([config]) + + context = CommandContext(cwd="", settings=mach.settings) + return mach.get_argument_parser(context) + + def test_command_aliases(self): + config = """ +[alias] +foo = cmd_foo +bar = cmd_bar +baz = cmd_bar --baz +cmd_bar = cmd_bar --baz +""" + parser = self.get_parser(config=config) + + args = parser.parse_args(["foo"]) + self.assertEqual(args.command, "cmd_foo") + + def assert_bar_baz(argv): + args = parser.parse_args(argv) + self.assertEqual(args.command, "cmd_bar") + self.assertTrue(args.command_args.baz) + + # The following should all result in |cmd_bar --baz| + assert_bar_baz(["bar", "--baz"]) + assert_bar_baz(["baz"]) + assert_bar_baz(["cmd_bar"]) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_entry_point.py b/python/mach/mach/test/test_entry_point.py new file mode 100644 index 0000000000..1129eba476 --- /dev/null +++ b/python/mach/mach/test/test_entry_point.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import imp +import sys +from pathlib import Path +from unittest.mock import patch + +from mozunit import main + +from mach.base import MachError +from mach.test.conftest import TestBase + + +class Entry: + """Stub replacement for pkg_resources.EntryPoint""" + + def __init__(self, providers): + self.providers = providers + + def load(self): + def _providers(): + return self.providers + + return _providers + + +class TestEntryPoints(TestBase): + """Test integrating with setuptools entry points""" + + provider_dir = Path(__file__).parent.resolve() / "providers" + + def _run_help(self): + return self._run_mach(["help"], entry_point="mach.providers") + + @patch("pkg_resources.iter_entry_points") + def test_load_entry_point_from_directory(self, mock): + # Ensure parent module is present otherwise we'll (likely) get + # an error due to unknown parent. + if "mach.commands" not in sys.modules: + mod = imp.new_module("mach.commands") + sys.modules["mach.commands"] = mod + + mock.return_value = [Entry([self.provider_dir])] + # Mach error raised due to conditions_invalid.py + with self.assertRaises(MachError): + self._run_help() + + @patch("pkg_resources.iter_entry_points") + def test_load_entry_point_from_file(self, mock): + mock.return_value = [Entry([self.provider_dir / "basic.py"])] + + result, stdout, stderr = self._run_help() + self.assertIsNone(result) + self.assertIn("cmd_foo", stdout) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_error_output.py b/python/mach/mach/test/test_error_output.py new file mode 100644 index 0000000000..12eab65856 --- /dev/null +++ b/python/mach/mach/test/test_error_output.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pathlib import Path + +from mozunit import main + +from mach.main import COMMAND_ERROR_TEMPLATE, MODULE_ERROR_TEMPLATE + + +def test_command_error(run_mach): + result, stdout, stderr = run_mach( + ["throw", "--message", "Command Error"], provider_files=Path("throw.py") + ) + assert result == 1 + assert COMMAND_ERROR_TEMPLATE % "throw" in stdout + + +def test_invoked_error(run_mach): + result, stdout, stderr = run_mach( + ["throw_deep", "--message", "Deep stack"], provider_files=Path("throw.py") + ) + assert result == 1 + assert MODULE_ERROR_TEMPLATE % "throw_deep" in stdout + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_logger.py b/python/mach/mach/test/test_logger.py new file mode 100644 index 0000000000..643d890de8 --- /dev/null +++ b/python/mach/mach/test/test_logger.py @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import time +import unittest + +from mozunit import main + +from mach.logging import StructuredHumanFormatter + + +class DummyLogger(logging.Logger): + def __init__(self, cb): + logging.Logger.__init__(self, "test") + + self._cb = cb + + def handle(self, record): + self._cb(record) + + +class TestStructuredHumanFormatter(unittest.TestCase): + def test_non_ascii_logging(self): + # Ensures the formatter doesn't choke when non-ASCII characters are + # present in printed parameters. + formatter = StructuredHumanFormatter(time.time()) + + def on_record(record): + result = formatter.format(record) + relevant = result[9:] + + self.assertEqual(relevant, "Test: s\xe9curit\xe9") + + logger = DummyLogger(on_record) + + value = "s\xe9curit\xe9" + + logger.log( + logging.INFO, + "Test: {utf}", + extra={"action": "action", "params": {"utf": value}}, + ) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_mach.py b/python/mach/mach/test/test_mach.py new file mode 100644 index 0000000000..38379d1b49 --- /dev/null +++ b/python/mach/mach/test/test_mach.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozunit import main + + +def test_set_isatty_environ(monkeypatch, get_mach): + # Make sure the 'MACH_STDOUT_ISATTY' variable gets set. + monkeypatch.delenv("MACH_STDOUT_ISATTY", raising=False) + monkeypatch.setattr(os, "isatty", lambda fd: True) + + m = get_mach() + orig_run = m._run + env_is_set = [] + + def wrap_run(*args, **kwargs): + env_is_set.append("MACH_STDOUT_ISATTY" in os.environ) + return orig_run(*args, **kwargs) + + monkeypatch.setattr(m, "_run", wrap_run) + + ret = m.run([]) + assert ret == 0 + assert env_is_set[0] + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_site.py b/python/mach/mach/test/test_site.py new file mode 100644 index 0000000000..d7c3d8c489 --- /dev/null +++ b/python/mach/mach/test/test_site.py @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from unittest import mock + +import pytest +from buildconfig import topsrcdir +from mozunit import main + +from mach.site import ( + PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS, + SitePackagesSource, + resolve_requirements, +) + + +@pytest.mark.parametrize( + "env_native_package_source,env_use_system_python,env_moz_automation,expected", + [ + ("system", False, False, SitePackagesSource.SYSTEM), + ("pip", False, False, SitePackagesSource.VENV), + ("none", False, False, SitePackagesSource.NONE), + (None, False, False, SitePackagesSource.VENV), + (None, False, True, SitePackagesSource.NONE), + (None, True, False, SitePackagesSource.NONE), + (None, True, True, SitePackagesSource.NONE), + ], +) +def test_resolve_package_source( + env_native_package_source, env_use_system_python, env_moz_automation, expected +): + with mock.patch.dict( + os.environ, + { + "MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE": env_native_package_source or "", + "MACH_USE_SYSTEM_PYTHON": "1" if env_use_system_python else "", + "MOZ_AUTOMATION": "1" if env_moz_automation else "", + }, + ): + assert SitePackagesSource.for_mach() == expected + + +def test_all_restricted_sites_dont_have_pypi_requirements(): + for site_name in PIP_NETWORK_INSTALL_RESTRICTED_VIRTUALENVS: + requirements = resolve_requirements(topsrcdir, site_name) + assert not requirements.pypi_requirements, ( + 'Sites that must be able to operate without "pip install" must not have any ' + f'mandatory "pypi requirements". However, the "{site_name}" site depends on: ' + f"{requirements.pypi_requirements}" + ) + + +if __name__ == "__main__": + main() diff --git a/python/mach/mach/test/test_site_activation.py b/python/mach/mach/test/test_site_activation.py new file mode 100644 index 0000000000..e034a27b76 --- /dev/null +++ b/python/mach/mach/test/test_site_activation.py @@ -0,0 +1,463 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import ast +import functools +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from subprocess import CompletedProcess +from typing import List + +import buildconfig +import mozunit +import pkg_resources +import pytest + +from mach.site import MozSiteMetadata, PythonVirtualenv, activate_virtualenv + + +class ActivationContext: + def __init__( + self, + topsrcdir: Path, + work_dir: Path, + original_python_path: str, + stdlib_paths: List[Path], + system_paths: List[Path], + required_mach_sys_paths: List[Path], + mach_requirement_paths: List[Path], + command_requirement_path: Path, + ): + self.topsrcdir = topsrcdir + self.work_dir = work_dir + self.original_python_path = original_python_path + self.stdlib_paths = stdlib_paths + self.system_paths = system_paths + self.required_moz_init_sys_paths = required_mach_sys_paths + self.mach_requirement_paths = mach_requirement_paths + self.command_requirement_path = command_requirement_path + + def virtualenv(self, name: str) -> PythonVirtualenv: + base_path = self.work_dir + + if name == "mach": + base_path = base_path / "_virtualenvs" + return PythonVirtualenv(str(base_path / name)) + + +def test_new_package_appears_in_pkg_resources(): + try: + # "carrot" was chosen as the package to use because: + # * It has to be a package that doesn't exist in-scope at the start (so, + # all vendored modules included in the test virtualenv aren't usage). + # * It must be on our internal PyPI mirror. + # Of the options, "carrot" is a small install that fits these requirements. + pkg_resources.get_distribution("carrot") + assert False, "Expected to not find 'carrot' as the initial state of the test" + except pkg_resources.DistributionNotFound: + pass + + with tempfile.TemporaryDirectory() as venv_dir: + subprocess.check_call( + [ + sys.executable, + "-m", + "venv", + venv_dir, + ] + ) + + venv = PythonVirtualenv(venv_dir) + venv.pip_install(["carrot==0.10.7"]) + + initial_metadata = MozSiteMetadata.from_runtime() + try: + metadata = MozSiteMetadata(None, None, None, None, venv.prefix) + with metadata.update_current_site(venv.python_path): + activate_virtualenv(venv) + + assert pkg_resources.get_distribution("carrot").version == "0.10.7" + finally: + MozSiteMetadata.current = initial_metadata + + +def test_sys_path_source_none_build(context): + original, mach, command = _run_activation_script_for_paths(context, "none", "build") + _assert_original_python_sys_path(context, original) + + assert not os.path.exists(context.virtualenv("mach").prefix) + assert mach == [ + *context.stdlib_paths, + *context.mach_requirement_paths, + ] + + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + context.command_requirement_path, + ] + assert command == expected_command_paths + + command_venv = _sys_path_of_virtualenv(context.virtualenv("build")) + assert command_venv == [Path(""), *expected_command_paths] + + +def test_sys_path_source_none_other(context): + original, mach, command = _run_activation_script_for_paths(context, "none", "other") + _assert_original_python_sys_path(context, original) + + assert not os.path.exists(context.virtualenv("mach").prefix) + assert mach == [ + *context.stdlib_paths, + *context.mach_requirement_paths, + ] + + command_virtualenv = PythonVirtualenv(str(context.work_dir / "other")) + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + context.command_requirement_path, + *(Path(p) for p in command_virtualenv.site_packages_dirs()), + ] + assert command == expected_command_paths + + command_venv = _sys_path_of_virtualenv(context.virtualenv("other")) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_venv_build(context): + original, mach, command = _run_activation_script_for_paths(context, "pip", "build") + _assert_original_python_sys_path(context, original) + + mach_virtualenv = context.virtualenv("mach") + expected_mach_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *(Path(p) for p in mach_virtualenv.site_packages_dirs()), + ] + assert mach == expected_mach_paths + + command_virtualenv = context.virtualenv("build") + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *(Path(p) for p in mach_virtualenv.site_packages_dirs()), + context.command_requirement_path, + *(Path(p) for p in command_virtualenv.site_packages_dirs()), + ] + assert command == expected_command_paths + + mach_venv = _sys_path_of_virtualenv(mach_virtualenv) + assert mach_venv == [ + Path(""), + *expected_mach_paths, + ] + + command_venv = _sys_path_of_virtualenv(command_virtualenv) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_venv_other(context): + original, mach, command = _run_activation_script_for_paths(context, "pip", "other") + _assert_original_python_sys_path(context, original) + + mach_virtualenv = context.virtualenv("mach") + expected_mach_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *(Path(p) for p in mach_virtualenv.site_packages_dirs()), + ] + assert mach == expected_mach_paths + + command_virtualenv = context.virtualenv("other") + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *(Path(p) for p in mach_virtualenv.site_packages_dirs()), + context.command_requirement_path, + *(Path(p) for p in command_virtualenv.site_packages_dirs()), + ] + assert command == expected_command_paths + + mach_venv = _sys_path_of_virtualenv(mach_virtualenv) + assert mach_venv == [ + Path(""), + *expected_mach_paths, + ] + + command_venv = _sys_path_of_virtualenv(command_virtualenv) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_system_build(context): + original, mach, command = _run_activation_script_for_paths( + context, "system", "build" + ) + _assert_original_python_sys_path(context, original) + + assert not os.path.exists(context.virtualenv("mach").prefix) + expected_mach_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *context.system_paths, + ] + assert mach == expected_mach_paths + + command_virtualenv = context.virtualenv("build") + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *context.system_paths, + context.command_requirement_path, + ] + assert command == expected_command_paths + + command_venv = _sys_path_of_virtualenv(command_virtualenv) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_system_other(context): + result = _run_activation_script( + context, + "system", + "other", + context.original_python_path, + stderr=subprocess.PIPE, + ) + assert result.returncode != 0 + assert ( + 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any sites ' + "other than" in result.stderr + ) + + +def test_sys_path_source_venvsystem_build(context): + venv_system_python = _create_venv_system_python( + context.work_dir, context.original_python_path + ) + venv_system_site_packages_dirs = [ + Path(p) for p in venv_system_python.site_packages_dirs() + ] + original, mach, command = _run_activation_script_for_paths( + context, "system", "build", venv_system_python.python_path + ) + + assert original == [ + Path(__file__).parent, + *context.required_moz_init_sys_paths, + *context.stdlib_paths, + *venv_system_site_packages_dirs, + ] + + assert not os.path.exists(context.virtualenv("mach").prefix) + expected_mach_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *venv_system_site_packages_dirs, + ] + assert mach == expected_mach_paths + + command_virtualenv = context.virtualenv("build") + expected_command_paths = [ + *context.stdlib_paths, + *context.mach_requirement_paths, + *venv_system_site_packages_dirs, + context.command_requirement_path, + ] + assert command == expected_command_paths + + command_venv = _sys_path_of_virtualenv(command_virtualenv) + assert command_venv == [ + Path(""), + *expected_command_paths, + ] + + +def test_sys_path_source_venvsystem_other(context): + venv_system_python = _create_venv_system_python( + context.work_dir, context.original_python_path + ) + result = _run_activation_script( + context, + "system", + "other", + venv_system_python.python_path, + stderr=subprocess.PIPE, + ) + assert result.returncode != 0 + assert ( + 'Cannot use MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE="system" for any sites ' + "other than" in result.stderr + ) + + +@pytest.fixture(name="context") +def _activation_context(): + original_python_path, stdlib_paths, system_paths = _original_python() + topsrcdir = Path(buildconfig.topsrcdir) + required_mach_sys_paths = [ + topsrcdir / "python" / "mach", + topsrcdir / "third_party" / "python" / "packaging", + topsrcdir / "third_party" / "python" / "pyparsing", + topsrcdir / "third_party" / "python" / "pip", + ] + + with tempfile.TemporaryDirectory() as work_dir: + # Get "resolved" version of path to ease comparison against "site"-added sys.path + # entries, as "site" calculates the realpath of provided locations. + work_dir = Path(work_dir).resolve() + mach_requirement_paths = [ + *required_mach_sys_paths, + work_dir / "mach_site_path", + ] + command_requirement_path = work_dir / "command_site_path" + (work_dir / "mach_site_path").touch() + command_requirement_path.touch() + yield ActivationContext( + topsrcdir, + work_dir, + original_python_path, + stdlib_paths, + system_paths, + required_mach_sys_paths, + mach_requirement_paths, + command_requirement_path, + ) + + +@functools.lru_cache(maxsize=None) +def _original_python(): + current_site = MozSiteMetadata.from_runtime() + stdlib_paths, system_paths = current_site.original_python.sys_path() + stdlib_paths = [Path(path) for path in _filter_pydev_from_paths(stdlib_paths)] + system_paths = [Path(path) for path in system_paths] + return current_site.original_python.python_path, stdlib_paths, system_paths + + +def _run_activation_script( + context: ActivationContext, + source: str, + site_name: str, + invoking_python: str, + **kwargs +) -> CompletedProcess: + return subprocess.run( + [ + invoking_python, + str(Path(__file__).parent / "script_site_activation.py"), + ], + stdout=subprocess.PIPE, + universal_newlines=True, + env={ + "TOPSRCDIR": str(context.topsrcdir), + "COMMAND_SITE": site_name, + "PYTHONPATH": os.pathsep.join( + str(p) for p in context.required_moz_init_sys_paths + ), + "MACH_SITE_PTH_REQUIREMENTS": os.pathsep.join( + str(p) for p in context.mach_requirement_paths + ), + "COMMAND_SITE_PTH_REQUIREMENTS": str(context.command_requirement_path), + "MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE": source, + "WORK_DIR": str(context.work_dir), + # These two variables are needed on Windows so that Python initializes + # properly and adds the "user site packages" to the sys.path like normal. + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + "APPDATA": os.environ.get("APPDATA", ""), + }, + **kwargs, + ) + + +def _run_activation_script_for_paths( + context: ActivationContext, source: str, site_name: str, invoking_python: str = None +) -> List[List[Path]]: + """Return the states of the sys.path when activating Mach-managed sites + + Three sys.path states are returned: + * The initial sys.path, equivalent to "path_to_python -c "import sys; print(sys.path)" + * The sys.path after activating the Mach site + * The sys.path after activating the command site + """ + + output = _run_activation_script( + context, + source, + site_name, + invoking_python or context.original_python_path, + check=True, + ).stdout + # Filter to the last line, which will have our nested list that we want to + # parse. This will avoid unrelated output, such as from virtualenv creation + output = output.splitlines()[-1] + return [ + [Path(path) for path in _filter_pydev_from_paths(paths)] + for paths in ast.literal_eval(output) + ] + + +def _assert_original_python_sys_path(context: ActivationContext, original: List[Path]): + # Assert that initial sys.path (prior to any activations) matches expectations. + assert original == [ + Path(__file__).parent, + *context.required_moz_init_sys_paths, + *context.stdlib_paths, + *context.system_paths, + ] + + +def _sys_path_of_virtualenv(virtualenv: PythonVirtualenv) -> List[Path]: + output = subprocess.run( + [virtualenv.python_path, "-c", "import sys; print(sys.path)"], + stdout=subprocess.PIPE, + universal_newlines=True, + env={ + # Needed for python to initialize properly + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + }, + check=True, + ).stdout + return [Path(path) for path in _filter_pydev_from_paths(ast.literal_eval(output))] + + +def _filter_pydev_from_paths(paths: List[str]) -> List[str]: + # Filter out injected "pydev" debugging tool if running within a JetBrains + # debugging context. + return [path for path in paths if "pydev" not in path and "JetBrains" not in path] + + +def _create_venv_system_python( + work_dir: Path, invoking_python: str +) -> PythonVirtualenv: + virtualenv = PythonVirtualenv(str(work_dir / "system_python")) + subprocess.run( + [ + invoking_python, + "-m", + "venv", + virtualenv.prefix, + "--without-pip", + ], + check=True, + ) + return virtualenv + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mach/mach/test/test_site_compatibility.py b/python/mach/mach/test/test_site_compatibility.py new file mode 100644 index 0000000000..4c1b6d5efa --- /dev/null +++ b/python/mach/mach/test/test_site_compatibility.py @@ -0,0 +1,189 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import shutil +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import mozunit +from buildconfig import topsrcdir + +from mach.requirements import MachEnvRequirements +from mach.site import PythonVirtualenv + + +def _resolve_command_site_names(): + site_names = [] + for child in (Path(topsrcdir) / "python" / "sites").iterdir(): + if not child.is_file(): + continue + + if child.suffix != ".txt": + continue + + if child.name == "mach.txt": + continue + + site_names.append(child.stem) + return site_names + + +def _requirement_definition_to_pip_format(site_name, cache, is_mach_or_build_env): + """Convert from parsed requirements object to pip-consumable format""" + requirements_path = Path(topsrcdir) / "python" / "sites" / f"{site_name}.txt" + requirements = MachEnvRequirements.from_requirements_definition( + topsrcdir, False, not is_mach_or_build_env, requirements_path + ) + + lines = [] + for pypi in ( + requirements.pypi_requirements + requirements.pypi_optional_requirements + ): + lines.append(str(pypi.requirement)) + + for vendored in requirements.vendored_requirements: + lines.append(str(cache.package_for_vendor_dir(Path(vendored.path)))) + + for pth in requirements.pth_requirements: + path = Path(pth.path) + + if "third_party" not in (p.name for p in path.parents): + continue + + for child in path.iterdir(): + if child.name.endswith(".dist-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has a ' + '".dist-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + if child.name.endswith(".egg-info"): + raise Exception( + f'In {requirements_path}, the "pth:" pointing to "{path}" has an ' + '".egg-info" file.\n' + 'Perhaps it should change to start with "vendored:" instead of ' + '"pth:".' + ) + + return "\n".join(lines) + + +class PackageCache: + def __init__(self, storage_dir: Path): + self._cache = {} + self._storage_dir = storage_dir + + def package_for_vendor_dir(self, vendor_path: Path): + if vendor_path in self._cache: + return self._cache[vendor_path] + + if not any((p for p in vendor_path.iterdir() if p.name.endswith(".dist-info"))): + # This vendored package is not a wheel. It may be a source package (with + # a setup.py), or just some Python code that was manually copied into the + # tree. If it's a source package, the setup.py file may be up a few levels + # from the referenced Python module path. + package_dir = vendor_path + while True: + if (package_dir / "setup.py").exists(): + break + elif package_dir.parent == package_dir: + raise Exception( + f'Package "{vendor_path}" is not a wheel and does not have a ' + 'setup.py file. Perhaps it should be "pth:" instead of ' + '"vendored:"?' + ) + package_dir = package_dir.parent + + self._cache[vendor_path] = package_dir + return package_dir + + # Pip requires that wheels have a version number in their name, even if + # it ignores it. We should parse out the version and put it in here + # so that failure debugging is easier, but that's non-trivial work. + # So, this "0" satisfies pip's naming requirement while being relatively + # obvious that it's a placeholder. + output_path = self._storage_dir / f"{vendor_path.name}-0-py3-none-any" + shutil.make_archive(str(output_path), "zip", vendor_path) + + whl_path = output_path.parent / (output_path.name + ".whl") + (output_path.parent / (output_path.name + ".zip")).rename(whl_path) + self._cache[vendor_path] = whl_path + + return whl_path + + +def test_sites_compatible(tmpdir: str): + command_site_names = _resolve_command_site_names() + work_dir = Path(tmpdir) + cache = PackageCache(work_dir) + mach_requirements = _requirement_definition_to_pip_format("mach", cache, True) + + # Create virtualenv to try to install all dependencies into. + virtualenv = PythonVirtualenv(str(work_dir / "env")) + subprocess.check_call( + [ + sys.executable, + "-m", + "venv", + "--without-pip", + virtualenv.prefix, + ] + ) + platlib_dir = virtualenv.resolve_sysconfig_packages_path("platlib") + third_party = Path(topsrcdir) / "third_party" / "python" + with open(os.path.join(platlib_dir, "site.pth"), "w") as pthfile: + pthfile.write( + "\n".join( + [ + str(third_party / "pip"), + str(third_party / "wheel"), + str(third_party / "setuptools"), + ] + ) + ) + + for name in command_site_names: + print(f'Checking compatibility of "{name}" site') + command_requirements = _requirement_definition_to_pip_format( + name, cache, name == "build" + ) + with open(work_dir / "requirements.txt", "w") as requirements_txt: + requirements_txt.write(mach_requirements) + requirements_txt.write("\n") + requirements_txt.write(command_requirements) + + # Attempt to install combined set of dependencies (global Mach + current + # command) + proc = subprocess.run( + [ + virtualenv.python_path, + "-m", + "pip", + "install", + "-r", + str(work_dir / "requirements.txt"), + ], + cwd=topsrcdir, + ) + if proc.returncode != 0: + print( + dedent( + f""" + Error: The '{name}' site contains dependencies that are not + compatible with the 'mach' site. Check the following files for + any conflicting packages mentioned in the prior error message: + + python/sites/mach.txt + python/sites/{name}.txt + """ + ) + ) + assert False + + +if __name__ == "__main__": + mozunit.main() diff --git a/python/mach/mach/test/zero_microseconds.py b/python/mach/mach/test/zero_microseconds.py new file mode 100644 index 0000000000..b1d523071f --- /dev/null +++ b/python/mach/mach/test/zero_microseconds.py @@ -0,0 +1,12 @@ +# This code is loaded via `mach python --exec-file`, so it runs in the scope of +# the `mach python` command. +old = self._mach_context.post_dispatch_handler # noqa: F821 + + +def handler(context, handler, instance, result, start_time, end_time, depth, args): + global old + # Round off sub-second precision. + old(context, handler, instance, result, int(start_time), end_time, depth, args) + + +self._mach_context.post_dispatch_handler = handler # noqa: F821 |