summaryrefslogtreecommitdiffstats
path: root/python/mozbuild/mozbuild/configure
diff options
context:
space:
mode:
Diffstat (limited to 'python/mozbuild/mozbuild/configure')
-rw-r--r--python/mozbuild/mozbuild/configure/__init__.py1311
-rw-r--r--python/mozbuild/mozbuild/configure/check_debug_ranges.py68
-rw-r--r--python/mozbuild/mozbuild/configure/constants.py131
-rw-r--r--python/mozbuild/mozbuild/configure/help.py90
-rw-r--r--python/mozbuild/mozbuild/configure/lint.py348
-rw-r--r--python/mozbuild/mozbuild/configure/options.py614
-rw-r--r--python/mozbuild/mozbuild/configure/util.py235
7 files changed, 2797 insertions, 0 deletions
diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py
new file mode 100644
index 0000000000..f60f179d6b
--- /dev/null
+++ b/python/mozbuild/mozbuild/configure/__init__.py
@@ -0,0 +1,1311 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import codecs
+import inspect
+import logging
+import os
+import re
+import sys
+import types
+from collections import OrderedDict
+from contextlib import contextmanager
+from functools import wraps
+
+import mozpack.path as mozpath
+import six
+from six.moves import builtins as __builtin__
+
+from mozbuild.configure.help import HelpFormatter
+from mozbuild.configure.options import (
+ HELP_OPTIONS_CATEGORY,
+ CommandLineHelper,
+ ConflictingOptionError,
+ InvalidOptionError,
+ Option,
+ OptionValue,
+)
+from mozbuild.configure.util import ConfigureOutputHandler, LineIO, getpreferredencoding
+from mozbuild.util import (
+ ReadOnlyDict,
+ ReadOnlyNamespace,
+ exec_,
+ memoize,
+ memoized_property,
+ system_encoding,
+)
+
+# TRACE logging level, below (thus more verbose than) DEBUG
+TRACE = 5
+
+
+class ConfigureError(Exception):
+ pass
+
+
+class SandboxDependsFunction(object):
+ """Sandbox-visible representation of @depends functions."""
+
+ def __init__(self, unsandboxed):
+ self._or = unsandboxed.__or__
+ self._and = unsandboxed.__and__
+ self._getattr = unsandboxed.__getattr__
+
+ def __call__(self, *arg, **kwargs):
+ raise ConfigureError("The `%s` function may not be called" % self.__name__)
+
+ def __or__(self, other):
+ if not isinstance(other, SandboxDependsFunction):
+ raise ConfigureError(
+ "Can only do binary arithmetic operations "
+ "with another @depends function."
+ )
+ return self._or(other).sandboxed
+
+ def __and__(self, other):
+ if not isinstance(other, SandboxDependsFunction):
+ raise ConfigureError(
+ "Can only do binary arithmetic operations "
+ "with another @depends function."
+ )
+ return self._and(other).sandboxed
+
+ def __cmp__(self, other):
+ raise ConfigureError("Cannot compare @depends functions.")
+
+ def __eq__(self, other):
+ raise ConfigureError("Cannot compare @depends functions.")
+
+ def __hash__(self):
+ return object.__hash__(self)
+
+ def __ne__(self, other):
+ raise ConfigureError("Cannot compare @depends functions.")
+
+ def __lt__(self, other):
+ raise ConfigureError("Cannot compare @depends functions.")
+
+ def __le__(self, other):
+ raise ConfigureError("Cannot compare @depends functions.")
+
+ def __gt__(self, other):
+ raise ConfigureError("Cannot compare @depends functions.")
+
+ def __ge__(self, other):
+ raise ConfigureError("Cannot compare @depends functions.")
+
+ def __getattr__(self, key):
+ return self._getattr(key).sandboxed
+
+ def __nonzero__(self):
+ raise ConfigureError("Cannot do boolean operations on @depends functions.")
+
+
+class DependsFunction(object):
+ __slots__ = (
+ "_func",
+ "_name",
+ "dependencies",
+ "when",
+ "sandboxed",
+ "sandbox",
+ "_result",
+ )
+
+ def __init__(self, sandbox, func, dependencies, when=None):
+ assert isinstance(sandbox, ConfigureSandbox)
+ assert not inspect.isgeneratorfunction(func)
+ # Allow non-functions when there are no dependencies. This is equivalent
+ # to passing a lambda that returns the given value.
+ if not (inspect.isroutine(func) or not dependencies):
+ print(func)
+ assert inspect.isroutine(func) or not dependencies
+ self._func = func
+ self._name = getattr(func, "__name__", None)
+ self.dependencies = dependencies
+ self.sandboxed = wraps(func)(SandboxDependsFunction(self))
+ self.sandbox = sandbox
+ self.when = when
+ sandbox._depends[self.sandboxed] = self
+
+ # Only @depends functions with a dependency on '--help' are executed
+ # immediately. Everything else is queued for later execution.
+ if sandbox._help_option in dependencies:
+ sandbox._value_for(self)
+ elif not sandbox._help:
+ sandbox._execution_queue.append((sandbox._value_for, (self,)))
+
+ @property
+ def name(self):
+ return self._name
+
+ @name.setter
+ def name(self, value):
+ self._name = value
+
+ @property
+ def sandboxed_dependencies(self):
+ return [
+ d.sandboxed if isinstance(d, DependsFunction) else d
+ for d in self.dependencies
+ ]
+
+ @memoize
+ def result(self):
+ if self.when and not self.sandbox._value_for(self.when):
+ return None
+
+ if inspect.isroutine(self._func):
+ resolved_args = [self.sandbox._value_for(d) for d in self.dependencies]
+ return self._func(*resolved_args)
+ return self._func
+
+ def __repr__(self):
+ return "<%s %s(%s)>" % (
+ self.__class__.__name__,
+ self.name,
+ ", ".join(repr(d) for d in self.dependencies),
+ )
+
+ def __or__(self, other):
+ if isinstance(other, SandboxDependsFunction):
+ other = self.sandbox._depends.get(other)
+ assert isinstance(other, DependsFunction)
+ assert self.sandbox is other.sandbox
+ return CombinedDependsFunction(self.sandbox, self.or_impl, (self, other))
+
+ @staticmethod
+ def or_impl(iterable):
+ # Applies "or" to all the items of iterable.
+ # e.g. if iterable contains a, b and c, returns `a or b or c`.
+ for i in iterable:
+ if i:
+ return i
+ return i
+
+ def __and__(self, other):
+ if isinstance(other, SandboxDependsFunction):
+ other = self.sandbox._depends.get(other)
+ assert isinstance(other, DependsFunction)
+ assert self.sandbox is other.sandbox
+ return CombinedDependsFunction(self.sandbox, self.and_impl, (self, other))
+
+ @staticmethod
+ def and_impl(iterable):
+ # Applies "and" to all the items of iterable.
+ # e.g. if iterable contains a, b and c, returns `a and b and c`.
+ for i in iterable:
+ if not i:
+ return i
+ return i
+
+ def __getattr__(self, key):
+ if key.startswith("_"):
+ return super(DependsFunction, self).__getattr__(key)
+ # Our function may return None or an object that simply doesn't have
+ # the wanted key. In that case, just return None.
+ return TrivialDependsFunction(
+ self.sandbox, lambda x: getattr(x, key, None), [self], self.when
+ )
+
+
+class TrivialDependsFunction(DependsFunction):
+ """Like a DependsFunction, but the linter won't expect it to have a
+ dependency on --help ever."""
+
+
+class CombinedDependsFunction(DependsFunction):
+ def __init__(self, sandbox, func, dependencies):
+ flatten_deps = []
+ for d in dependencies:
+ if isinstance(d, CombinedDependsFunction) and d._func is func:
+ for d2 in d.dependencies:
+ if d2 not in flatten_deps:
+ flatten_deps.append(d2)
+ elif d not in flatten_deps:
+ flatten_deps.append(d)
+
+ super(CombinedDependsFunction, self).__init__(sandbox, func, flatten_deps)
+
+ @memoize
+ def result(self):
+ resolved_args = (self.sandbox._value_for(d) for d in self.dependencies)
+ return self._func(resolved_args)
+
+ def __eq__(self, other):
+ return (
+ isinstance(other, self.__class__)
+ and self._func is other._func
+ and set(self.dependencies) == set(other.dependencies)
+ )
+
+ def __hash__(self):
+ return object.__hash__(self)
+
+ def __ne__(self, other):
+ return not self == other
+
+
+class SandboxedGlobal(dict):
+ """Identifiable dict type for use as function global"""
+
+
+def forbidden_import(*args, **kwargs):
+ raise ImportError("Importing modules is forbidden")
+
+
+class ConfigureSandbox(dict):
+ """Represents a sandbox for executing Python code for build configuration.
+ This is a different kind of sandboxing than the one used for moz.build
+ processing.
+
+ The sandbox has 9 primitives:
+ - option
+ - depends
+ - template
+ - imports
+ - include
+ - set_config
+ - set_define
+ - imply_option
+ - only_when
+
+ `option`, `include`, `set_config`, `set_define` and `imply_option` are
+ functions. `depends`, `template`, and `imports` are decorators. `only_when`
+ is a context_manager.
+
+ These primitives are declared as name_impl methods to this class and
+ the mapping name -> name_impl is done automatically in __getitem__.
+
+ Additional primitives should be frowned upon to keep the sandbox itself as
+ simple as possible. Instead, helpers should be created within the sandbox
+ with the existing primitives.
+
+ The sandbox is given, at creation, a dict where the yielded configuration
+ will be stored.
+
+ config = {}
+ sandbox = ConfigureSandbox(config)
+ sandbox.run(path)
+ do_stuff(config)
+ """
+
+ # The default set of builtins. We expose unicode as str to make sandboxed
+ # files more python3-ready.
+ BUILTINS = ReadOnlyDict(
+ {
+ b: getattr(__builtin__, b, None)
+ for b in (
+ "AssertionError",
+ "False",
+ "None",
+ "True",
+ "__build_class__", # will be None on py2
+ "all",
+ "any",
+ "bool",
+ "dict",
+ "enumerate",
+ "getattr",
+ "hasattr",
+ "int",
+ "isinstance",
+ "len",
+ "list",
+ "max",
+ "min",
+ "range",
+ "set",
+ "sorted",
+ "tuple",
+ "zip",
+ )
+ },
+ __import__=forbidden_import,
+ str=six.text_type,
+ )
+
+ # Expose a limited set of functions from os.path
+ OS = ReadOnlyNamespace(
+ path=ReadOnlyNamespace(
+ **{
+ k: getattr(mozpath, k, getattr(os.path, k))
+ for k in (
+ "abspath",
+ "basename",
+ "dirname",
+ "isabs",
+ "join",
+ "normcase",
+ "normpath",
+ "realpath",
+ "relpath",
+ )
+ }
+ )
+ )
+
+ def __init__(
+ self,
+ config,
+ environ=os.environ,
+ argv=sys.argv,
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ logger=None,
+ ):
+ dict.__setitem__(self, "__builtins__", self.BUILTINS)
+
+ self._environ = dict(environ)
+
+ self._paths = []
+ self._all_paths = set()
+ self._templates = set()
+ # Associate SandboxDependsFunctions to DependsFunctions.
+ self._depends = OrderedDict()
+ self._seen = set()
+ # Store the @imports added to a given function.
+ self._imports = {}
+
+ self._options = OrderedDict()
+ # Store raw option (as per command line or environment) for each Option
+ self._raw_options = OrderedDict()
+
+ # Store options added with `imply_option`, and the reason they were
+ # added (which can either have been given to `imply_option`, or
+ # inferred. Their order matters, so use a list.
+ self._implied_options = []
+
+ # Store all results from _prepare_function
+ self._prepared_functions = set()
+
+ # Queue of functions to execute, with their arguments
+ self._execution_queue = []
+
+ # Store the `when`s associated to some options.
+ self._conditions = {}
+
+ # A list of conditions to apply as a default `when` for every *_impl()
+ self._default_conditions = []
+
+ self._helper = CommandLineHelper(environ, argv)
+
+ assert isinstance(config, dict)
+ self._config = config
+
+ # Tracks how many templates "deep" we are in the stack.
+ self._template_depth = 0
+
+ logging.addLevelName(TRACE, "TRACE")
+ if logger is None:
+ logger = moz_logger = logging.getLogger("moz.configure")
+ logger.setLevel(logging.DEBUG)
+ formatter = logging.Formatter("%(levelname)s: %(message)s")
+ handler = ConfigureOutputHandler(stdout, stderr)
+ handler.setFormatter(formatter)
+ queue_debug = handler.queue_debug
+ logger.addHandler(handler)
+
+ else:
+ assert isinstance(logger, logging.Logger)
+ moz_logger = None
+
+ @contextmanager
+ def queue_debug():
+ yield
+
+ self._logger = logger
+
+ # Some callers will manage to log a bytestring with characters in it
+ # that can't be converted to ascii. Make our log methods robust to this
+ # by detecting the encoding that a producer is likely to have used.
+ encoding = getpreferredencoding()
+
+ def wrapped_log_method(logger, key):
+ method = getattr(logger, key)
+
+ def wrapped(*args, **kwargs):
+ out_args = [
+ six.ensure_text(arg, encoding=encoding or "utf-8")
+ if isinstance(arg, six.binary_type)
+ else arg
+ for arg in args
+ ]
+ return method(*out_args, **kwargs)
+
+ return wrapped
+
+ log_namespace = {
+ k: wrapped_log_method(logger, k)
+ for k in ("debug", "info", "warning", "error")
+ }
+ log_namespace["queue_debug"] = queue_debug
+ self.log_impl = ReadOnlyNamespace(**log_namespace)
+
+ self._help = None
+ self._help_option = self.option_impl(
+ "--help", help="print this message", category=HELP_OPTIONS_CATEGORY
+ )
+ self._seen.add(self._help_option)
+
+ self._always = DependsFunction(self, lambda: True, [])
+ self._never = DependsFunction(self, lambda: False, [])
+
+ if self._value_for(self._help_option):
+ self._help = HelpFormatter(argv[0])
+ self._help.add(self._help_option)
+ elif moz_logger:
+ handler = logging.FileHandler(
+ "config.log", mode="w", delay=True, encoding="utf-8"
+ )
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+ def include_file(self, path):
+ """Include one file in the sandbox. Users of this class probably want
+ to use `run` instead.
+
+ Note: this will execute all template invocations, as well as @depends
+ functions that depend on '--help', but nothing else.
+ """
+
+ if self._paths:
+ path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
+ path = mozpath.normpath(path)
+ if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
+ raise ConfigureError(
+ "Cannot include `%s` because it is not in a subdirectory "
+ "of `%s`" % (path, mozpath.dirname(self._paths[0]))
+ )
+ else:
+ path = mozpath.realpath(mozpath.abspath(path))
+ if path in self._all_paths:
+ raise ConfigureError(
+ "Cannot include `%s` because it was included already." % path
+ )
+ self._paths.append(path)
+ self._all_paths.add(path)
+
+ with open(path, "rb") as fh:
+ source = fh.read()
+
+ code = compile(source, path, "exec")
+
+ exec_(code, self)
+
+ self._paths.pop(-1)
+
+ def run(self, path=None):
+ """Executes the given file within the sandbox, as well as everything
+ pending from any other included file, and ensure the overall
+ consistency of the executed script(s)."""
+ if path:
+ self.include_file(path)
+
+ for option in six.itervalues(self._options):
+ # All options must be referenced by some @depends function
+ if option not in self._seen:
+ raise ConfigureError(
+ "Option `%s` is not handled ; reference it with a @depends"
+ % option.option
+ )
+
+ self._value_for(option)
+
+ # All implied options should exist.
+ for implied_option in self._implied_options:
+ value = self._resolve(implied_option.value)
+ if value is not None:
+ # There are two ways to end up here: either the implied option
+ # is unknown, or it's known but there was a dependency loop
+ # that prevented the implication from being applied.
+ option = self._options.get(implied_option.name)
+ if not option:
+ raise ConfigureError(
+ "`%s`, emitted from `%s` line %d, is unknown."
+ % (
+ implied_option.option,
+ implied_option.caller[1],
+ implied_option.caller[2],
+ )
+ )
+ # If the option is known, check that the implied value doesn't
+ # conflict with what value was attributed to the option.
+ if implied_option.when and not self._value_for(implied_option.when):
+ continue
+ option_value = self._value_for_option(option)
+ if value != option_value:
+ reason = implied_option.reason
+ if isinstance(reason, Option):
+ reason = self._raw_options.get(reason) or reason.option
+ reason = reason.split("=", 1)[0]
+ value = OptionValue.from_(value)
+ raise InvalidOptionError(
+ "'%s' implied by '%s' conflicts with '%s' from the %s"
+ % (
+ value.format(option.option),
+ reason,
+ option_value.format(option.option),
+ option_value.origin,
+ )
+ )
+
+ # All options should have been removed (handled) by now.
+ for arg in self._helper:
+ without_value = arg.split("=", 1)[0]
+ msg = "Unknown option: %s" % without_value
+ if self._help:
+ self._logger.warning(msg)
+ else:
+ raise InvalidOptionError(msg)
+
+ # Run the execution queue
+ for func, args in self._execution_queue:
+ func(*args)
+
+ if self._help:
+ with LineIO(self.log_impl.info) as out:
+ self._help.usage(out)
+
+ def __getitem__(self, key):
+ impl = "%s_impl" % key
+ func = getattr(self, impl, None)
+ if func:
+ return func
+
+ return super(ConfigureSandbox, self).__getitem__(key)
+
+ def __setitem__(self, key, value):
+ if (
+ key in self.BUILTINS
+ or key == "__builtins__"
+ or hasattr(self, "%s_impl" % key)
+ ):
+ raise KeyError("Cannot reassign builtins")
+
+ if inspect.isfunction(value) and value not in self._templates:
+ value = self._prepare_function(value)
+
+ elif (
+ not isinstance(value, SandboxDependsFunction)
+ and value not in self._templates
+ and not (inspect.isclass(value) and issubclass(value, Exception))
+ ):
+ raise KeyError(
+ "Cannot assign `%s` because it is neither a "
+ "@depends nor a @template" % key
+ )
+
+ if isinstance(value, SandboxDependsFunction):
+ self._depends[value].name = key
+
+ return super(ConfigureSandbox, self).__setitem__(key, value)
+
+ def _resolve(self, arg):
+ if isinstance(arg, SandboxDependsFunction):
+ return self._value_for_depends(self._depends[arg])
+ return arg
+
+ def _value_for(self, obj):
+ if isinstance(obj, SandboxDependsFunction):
+ assert obj in self._depends
+ return self._value_for_depends(self._depends[obj])
+
+ elif isinstance(obj, DependsFunction):
+ return self._value_for_depends(obj)
+
+ elif isinstance(obj, Option):
+ return self._value_for_option(obj)
+
+ assert False
+
+ @memoize
+ def _value_for_depends(self, obj):
+ value = obj.result()
+ self._logger.log(TRACE, "%r = %r", obj, value)
+ return value
+
+ @memoize
+ def _value_for_option(self, option):
+ implied = {}
+ matching_implied_options = [
+ o for o in self._implied_options if o.name in (option.name, option.env)
+ ]
+ # Update self._implied_options before going into the loop with the non-matching
+ # options.
+ self._implied_options = [
+ o for o in self._implied_options if o.name not in (option.name, option.env)
+ ]
+
+ for implied_option in matching_implied_options:
+ if implied_option.when and not self._value_for(implied_option.when):
+ continue
+
+ value = self._resolve(implied_option.value)
+
+ if value is not None:
+ value = OptionValue.from_(value)
+ opt = value.format(implied_option.option)
+ self._helper.add(opt, "implied")
+ implied[opt] = implied_option
+
+ try:
+ value, option_string = self._helper.handle(option)
+ except ConflictingOptionError as e:
+ reason = implied[e.arg].reason
+ if isinstance(reason, Option):
+ reason = self._raw_options.get(reason) or reason.option
+ reason = reason.split("=", 1)[0]
+ raise InvalidOptionError(
+ "'%s' implied by '%s' conflicts with '%s' from the %s"
+ % (e.arg, reason, e.old_arg, e.old_origin)
+ )
+
+ if value.origin == "implied":
+ recursed_value = getattr(self, "__value_for_option").get((option,))
+ if recursed_value is not None:
+ _, filename, line, _, _, _ = implied[value.format(option.option)].caller
+ raise ConfigureError(
+ "'%s' appears somewhere in the direct or indirect dependencies when "
+ "resolving imply_option at %s:%d" % (option.option, filename, line)
+ )
+
+ if option_string:
+ self._raw_options[option] = option_string
+
+ when = self._conditions.get(option)
+ # If `when` resolves to a false-ish value, we always return None.
+ # This makes option(..., when='--foo') equivalent to
+ # option(..., when=depends('--foo')(lambda x: x)).
+ if when and not self._value_for(when) and value is not None:
+ # If the option was passed explicitly, we throw an error that
+ # the option is not available. Except when the option was passed
+ # from the environment, because that would be too cumbersome.
+ if value.origin not in ("default", "environment"):
+ raise InvalidOptionError(
+ "%s is not available in this configuration"
+ % option_string.split("=", 1)[0]
+ )
+ self._logger.log(TRACE, "%r = None", option)
+ return None
+
+ self._logger.log(TRACE, "%r = %r", option, value)
+ return value
+
+ def _dependency(self, arg, callee_name, arg_name=None):
+ if isinstance(arg, six.string_types):
+ prefix, name, values = Option.split_option(arg)
+ if values != ():
+ raise ConfigureError("Option must not contain an '='")
+ if name not in self._options:
+ raise ConfigureError(
+ "'%s' is not a known option. " "Maybe it's declared too late?" % arg
+ )
+ arg = self._options[name]
+ self._seen.add(arg)
+ elif isinstance(arg, SandboxDependsFunction):
+ assert arg in self._depends
+ arg = self._depends[arg]
+ else:
+ raise TypeError(
+ "Cannot use object of type '%s' as %sargument to %s"
+ % (
+ type(arg).__name__,
+ "`%s` " % arg_name if arg_name else "",
+ callee_name,
+ )
+ )
+ return arg
+
+ def _normalize_when(self, when, callee_name):
+ if when is True:
+ when = self._always
+ elif when is False:
+ when = self._never
+ elif when is not None:
+ when = self._dependency(when, callee_name, "when")
+
+ if self._default_conditions:
+ # Create a pseudo @depends function for the combination of all
+ # default conditions and `when`.
+ dependencies = [when] if when else []
+ dependencies.extend(self._default_conditions)
+ if len(dependencies) == 1:
+ return dependencies[0]
+ return CombinedDependsFunction(self, all, dependencies)
+ return when
+
+ @contextmanager
+ def only_when_impl(self, when):
+ """Implementation of only_when()
+
+ `only_when` is a context manager that essentially makes calls to
+ other sandbox functions within the context block ignored.
+ """
+ when = self._normalize_when(when, "only_when")
+ if when and self._default_conditions[-1:] != [when]:
+ self._default_conditions.append(when)
+ yield
+ self._default_conditions.pop()
+ else:
+ yield
+
+ def option_impl(self, *args, **kwargs):
+ """Implementation of option()
+ This function creates and returns an Option() object, passing it the
+ resolved arguments (uses the result of functions when functions are
+ passed). In most cases, the result of this function is not expected to
+ be used.
+ Command line argument/environment variable parsing for this Option is
+ handled here.
+ """
+ when = self._normalize_when(kwargs.get("when"), "option")
+ args = [self._resolve(arg) for arg in args]
+ kwargs = {k: self._resolve(v) for k, v in six.iteritems(kwargs) if k != "when"}
+ # The Option constructor needs to look up the stack to infer a category
+ # for the Option, since the category is based on the filename where the
+ # Option is defined. However, if the Option is defined in a template, we
+ # want the category to reference the caller of the template rather than
+ # the caller of the option() function.
+ kwargs["define_depth"] = self._template_depth * 3
+ option = Option(*args, **kwargs)
+ if when:
+ self._conditions[option] = when
+ if option.name in self._options:
+ raise ConfigureError("Option `%s` already defined" % option.option)
+ if option.env in self._options:
+ raise ConfigureError("Option `%s` already defined" % option.env)
+ if option.name:
+ self._options[option.name] = option
+ if option.env:
+ self._options[option.env] = option
+
+ if self._help and (when is None or self._value_for(when)):
+ self._help.add(option)
+
+ return option
+
+ def depends_impl(self, *args, **kwargs):
+ """Implementation of @depends()
+ This function is a decorator. It returns a function that subsequently
+ takes a function and returns a dummy function. The dummy function
+ identifies the actual function for the sandbox, while preventing
+ further function calls from within the sandbox.
+
+ @depends() takes a variable number of option strings or dummy function
+ references. The decorated function is called as soon as the decorator
+ is called, and the arguments it receives are the OptionValue or
+ function results corresponding to each of the arguments to @depends.
+ As an exception, when a HelpFormatter is attached, only functions that
+ have '--help' in their @depends argument list are called.
+
+ The decorated function is altered to use a different global namespace
+ for its execution. This different global namespace exposes a limited
+ set of functions from os.path.
+ """
+ for k in kwargs:
+ if k != "when":
+ raise TypeError(
+ "depends_impl() got an unexpected keyword argument '%s'" % k
+ )
+
+ when = self._normalize_when(kwargs.get("when"), "@depends")
+
+ if not when and not args:
+ raise ConfigureError("@depends needs at least one argument")
+
+ dependencies = tuple(self._dependency(arg, "@depends") for arg in args)
+
+ conditions = [
+ self._conditions[d]
+ for d in dependencies
+ if d in self._conditions and isinstance(d, Option)
+ ]
+ for c in conditions:
+ if c != when:
+ raise ConfigureError(
+ "@depends function needs the same `when` "
+ "as options it depends on"
+ )
+
+ def decorator(func):
+ if inspect.isgeneratorfunction(func):
+ raise ConfigureError(
+ "Cannot decorate generator functions with @depends"
+ )
+ if inspect.isroutine(func):
+ if func in self._templates:
+ raise TypeError("Cannot use a @template function here")
+ func = self._prepare_function(func)
+ elif isinstance(func, SandboxDependsFunction):
+ raise TypeError("Cannot nest @depends functions")
+ elif dependencies:
+ raise TypeError(
+ "Cannot wrap literal values in @depends with dependencies"
+ )
+ depends = DependsFunction(self, func, dependencies, when=when)
+ return depends.sandboxed
+
+ return decorator
+
+ def include_impl(self, what, when=None):
+ """Implementation of include().
+ Allows to include external files for execution in the sandbox.
+ It is possible to use a @depends function as argument, in which case
+ the result of the function is the file name to include. This latter
+ feature is only really meant for --enable-application/--enable-project.
+ """
+ with self.only_when_impl(when):
+ what = self._resolve(what)
+ if what:
+ if not isinstance(what, six.string_types):
+ raise TypeError("Unexpected type: '%s'" % type(what).__name__)
+ self.include_file(what)
+
+ def template_impl(self, func):
+ """Implementation of @template.
+ This function is a decorator. Template functions are called
+ immediately. They are altered so that their global namespace exposes
+ a limited set of functions from os.path, as well as `depends` and
+ `option`.
+ Templates allow to simplify repetitive constructs, or to implement
+ helper decorators and somesuch.
+ """
+
+ def update_globals(glob):
+ glob.update(
+ (k[: -len("_impl")], getattr(self, k))
+ for k in dir(self)
+ if k.endswith("_impl") and k != "template_impl"
+ )
+ glob.update((k, v) for k, v in six.iteritems(self) if k not in glob)
+
+ template = self._prepare_function(func, update_globals)
+
+ # Any function argument to the template must be prepared to be sandboxed.
+ # If the template itself returns a function (in which case, it's very
+ # likely a decorator), that function must be prepared to be sandboxed as
+ # well.
+ def wrap_template(template):
+ isfunction = inspect.isfunction
+
+ def maybe_prepare_function(obj):
+ if isfunction(obj):
+ return self._prepare_function(obj)
+ return obj
+
+ # The following function may end up being prepared to be sandboxed,
+ # so it mustn't depend on anything from the global scope in this
+ # file. It can however depend on variables from the closure, thus
+ # maybe_prepare_function and isfunction are declared above to be
+ # available there.
+ @self.wraps(template)
+ def wrapper(*args, **kwargs):
+ args = [maybe_prepare_function(arg) for arg in args]
+ kwargs = {k: maybe_prepare_function(v) for k, v in kwargs.items()}
+ self._template_depth += 1
+ ret = template(*args, **kwargs)
+ self._template_depth -= 1
+ if isfunction(ret):
+ # We can't expect the sandboxed code to think about all the
+ # details of implementing decorators, so do some of the
+ # work for them. If the function takes exactly one function
+ # as argument and returns a function, it must be a
+ # decorator, so mark the returned function as wrapping the
+ # function passed in.
+ if len(args) == 1 and not kwargs and isfunction(args[0]):
+ ret = self.wraps(args[0])(ret)
+ return wrap_template(ret)
+ return ret
+
+ return wrapper
+
+ wrapper = wrap_template(template)
+ self._templates.add(wrapper)
+ return wrapper
+
+ def wraps(self, func):
+ return wraps(func)
+
+ RE_MODULE = re.compile("^[a-zA-Z0-9_\.]+$")
+
+ def imports_impl(self, _import, _from=None, _as=None):
+ """Implementation of @imports.
+ This decorator imports the given _import from the given _from module
+ optionally under a different _as name.
+ The options correspond to the various forms for the import builtin.
+
+ @imports('sys')
+ @imports(_from='mozpack', _import='path', _as='mozpath')
+ """
+ for value, required in ((_import, True), (_from, False), (_as, False)):
+
+ if not isinstance(value, six.string_types) and (
+ required or value is not None
+ ):
+ raise TypeError("Unexpected type: '%s'" % type(value).__name__)
+ if value is not None and not self.RE_MODULE.match(value):
+ raise ValueError("Invalid argument to @imports: '%s'" % value)
+ if _as and "." in _as:
+ raise ValueError("Invalid argument to @imports: '%s'" % _as)
+
+ def decorator(func):
+ if func in self._templates:
+ raise ConfigureError("@imports must appear after @template")
+ if func in self._depends:
+ raise ConfigureError("@imports must appear after @depends")
+ # For the imports to apply in the order they appear in the
+ # .configure file, we accumulate them in reverse order and apply
+ # them later.
+ imports = self._imports.setdefault(func, [])
+ imports.insert(0, (_from, _import, _as))
+ return func
+
+ return decorator
+
+ def _apply_imports(self, func, glob):
+ for _from, _import, _as in self._imports.pop(func, ()):
+ self._get_one_import(_from, _import, _as, glob)
+
+ def _handle_wrapped_import(self, _from, _import, _as, glob):
+ """Given the name of a module, "import" a mocked package into the glob
+ iff the module is one that we wrap (either for the sandbox or for the
+ purpose of testing). Applies if the wrapped module is exposed by an
+ attribute of `self`.
+
+ For example, if the import statement is `from os import environ`, then
+ this function will set
+ glob['environ'] = self._wrapped_os.environ.
+
+ Iff this function handles the given import, return True.
+ """
+ module = (_from or _import).split(".")[0]
+ attr = "_wrapped_" + module
+ wrapped = getattr(self, attr, None)
+ if wrapped:
+ if _as or _from:
+ obj = self._recursively_get_property(
+ module, (_from + "." if _from else "") + _import, wrapped
+ )
+ glob[_as or _import] = obj
+ else:
+ glob[module] = wrapped
+ return True
+ else:
+ return False
+
+ def _recursively_get_property(self, module, what, wrapped):
+ """Traverse the wrapper object `wrapped` (which represents the module
+ `module`) and return the property represented by `what`, which may be a
+ series of nested attributes.
+
+ For example, if `module` is 'os' and `what` is 'os.path.join',
+ return `wrapped.path.join`.
+ """
+ if what == module:
+ return wrapped
+ assert what.startswith(module + ".")
+ attrs = what[len(module + ".") :].split(".")
+ for attr in attrs:
+ wrapped = getattr(wrapped, attr)
+ return wrapped
+
+ @memoized_property
+ def _wrapped_os(self):
+ wrapped_os = {}
+ exec_("from os import *", {}, wrapped_os)
+ # Special case os and os.environ so that os.environ is our copy of
+ # the environment.
+ wrapped_os["environ"] = self._environ
+ # Also override some os.path functions with ours.
+ wrapped_path = {}
+ exec_("from os.path import *", {}, wrapped_path)
+ wrapped_path.update(self.OS.path.__dict__)
+ wrapped_os["path"] = ReadOnlyNamespace(**wrapped_path)
+ return ReadOnlyNamespace(**wrapped_os)
+
+ @memoized_property
+ def _wrapped_subprocess(self):
+ wrapped_subprocess = {}
+ exec_("from subprocess import *", {}, wrapped_subprocess)
+
+ def wrap(function):
+ def wrapper(*args, **kwargs):
+ if kwargs.get("env") is None and self._environ:
+ kwargs["env"] = dict(self._environ)
+
+ return function(*args, **kwargs)
+
+ return wrapper
+
+ for f in ("call", "check_call", "check_output", "Popen", "run"):
+ # `run` is new to python 3.5. In case this still runs from python2
+ # code, avoid failing here.
+ if f in wrapped_subprocess:
+ wrapped_subprocess[f] = wrap(wrapped_subprocess[f])
+
+ return ReadOnlyNamespace(**wrapped_subprocess)
+
+ @memoized_property
+ def _wrapped_six(self):
+ if six.PY3:
+ return six
+ wrapped_six = {}
+ exec_("from six import *", {}, wrapped_six)
+ wrapped_six_moves = {}
+ exec_("from six.moves import *", {}, wrapped_six_moves)
+ wrapped_six_moves_builtins = {}
+ exec_("from six.moves.builtins import *", {}, wrapped_six_moves_builtins)
+
+ # Special case for the open() builtin, because otherwise, using it
+ # fails with "IOError: file() constructor not accessible in
+ # restricted mode". We also make open() look more like python 3's,
+ # decoding to unicode strings unless the mode says otherwise.
+ def wrapped_open(name, mode=None, buffering=None):
+ args = (name,)
+ kwargs = {}
+ if buffering is not None:
+ kwargs["buffering"] = buffering
+ if mode is not None:
+ args += (mode,)
+ if "b" in mode:
+ return open(*args, **kwargs)
+ kwargs["encoding"] = system_encoding
+ return codecs.open(*args, **kwargs)
+
+ wrapped_six_moves_builtins["open"] = wrapped_open
+ wrapped_six_moves["builtins"] = ReadOnlyNamespace(**wrapped_six_moves_builtins)
+ wrapped_six["moves"] = ReadOnlyNamespace(**wrapped_six_moves)
+
+ return ReadOnlyNamespace(**wrapped_six)
+
+ def _get_one_import(self, _from, _import, _as, glob):
+ """Perform the given import, placing the result into the dict glob."""
+ if not _from and _import == "__builtin__":
+ glob[_as or "__builtin__"] = __builtin__
+ return
+ if _from == "__builtin__":
+ _from = "six.moves.builtins"
+ # The special `__sandbox__` module gives access to the sandbox
+ # instance.
+ if not _from and _import == "__sandbox__":
+ glob[_as or _import] = self
+ return
+ if self._handle_wrapped_import(_from, _import, _as, glob):
+ return
+ # If we've gotten this far, we should just do a normal import.
+ # Until this proves to be a performance problem, just construct an
+ # import statement and execute it.
+ import_line = "%simport %s%s" % (
+ ("from %s " % _from) if _from else "",
+ _import,
+ (" as %s" % _as) if _as else "",
+ )
+ exec_(import_line, {}, glob)
+
+ def _resolve_and_set(self, data, name, value, when=None):
+ # Don't set anything when --help was on the command line
+ if self._help:
+ return
+ if when and not self._value_for(when):
+ return
+ name = self._resolve(name)
+ if name is None:
+ return
+ if not isinstance(name, six.string_types):
+ raise TypeError("Unexpected type: '%s'" % type(name).__name__)
+ if name in data:
+ raise ConfigureError(
+ "Cannot add '%s' to configuration: Key already " "exists" % name
+ )
+ value = self._resolve(value)
+ if value is not None:
+ if self._logger.isEnabledFor(TRACE):
+ if data is self._config:
+ self._logger.log(TRACE, "set_config(%s, %r)", name, value)
+ elif data is self._config.get("DEFINES"):
+ self._logger.log(TRACE, "set_define(%s, %r)", name, value)
+ data[name] = value
+
+ def set_config_impl(self, name, value, when=None):
+ """Implementation of set_config().
+ Set the configuration items with the given name to the given value.
+ Both `name` and `value` can be references to @depends functions,
+ in which case the result from these functions is used. If the result
+ of either function is None, the configuration item is not set.
+ """
+ when = self._normalize_when(when, "set_config")
+
+ self._execution_queue.append(
+ (self._resolve_and_set, (self._config, name, value, when))
+ )
+
+ def set_define_impl(self, name, value, when=None):
+ """Implementation of set_define().
+ Set the define with the given name to the given value. Both `name` and
+ `value` can be references to @depends functions, in which case the
+ result from these functions is used. If the result of either function
+ is None, the define is not set. If the result is False, the define is
+ explicitly undefined (-U).
+ """
+ when = self._normalize_when(when, "set_define")
+
+ defines = self._config.setdefault("DEFINES", {})
+ self._execution_queue.append(
+ (self._resolve_and_set, (defines, name, value, when))
+ )
+
+ def imply_option_impl(self, option, value, reason=None, when=None):
+ """Implementation of imply_option().
+ Injects additional options as if they had been passed on the command
+ line. The `option` argument is a string as in option()'s `name` or
+ `env`. The option must be declared after `imply_option` references it.
+ The `value` argument indicates the value to pass to the option.
+ It can be:
+ - True. In this case `imply_option` injects the positive option
+
+ (--enable-foo/--with-foo).
+ imply_option('--enable-foo', True)
+ imply_option('--disable-foo', True)
+
+ are both equivalent to `--enable-foo` on the command line.
+
+ - False. In this case `imply_option` injects the negative option
+
+ (--disable-foo/--without-foo).
+ imply_option('--enable-foo', False)
+ imply_option('--disable-foo', False)
+
+ are both equivalent to `--disable-foo` on the command line.
+
+ - None. In this case `imply_option` does nothing.
+ imply_option('--enable-foo', None)
+ imply_option('--disable-foo', None)
+
+ are both equivalent to not passing any flag on the command line.
+
+ - a string or a tuple. In this case `imply_option` injects the positive
+ option with the given value(s).
+
+ imply_option('--enable-foo', 'a')
+ imply_option('--disable-foo', 'a')
+
+ are both equivalent to `--enable-foo=a` on the command line.
+ imply_option('--enable-foo', ('a', 'b'))
+ imply_option('--disable-foo', ('a', 'b'))
+
+ are both equivalent to `--enable-foo=a,b` on the command line.
+
+ Because imply_option('--disable-foo', ...) can be misleading, it is
+ recommended to use the positive form ('--enable' or '--with') for
+ `option`.
+
+ The `value` argument can also be (and usually is) a reference to a
+ @depends function, in which case the result of that function will be
+ used as per the descripted mapping above.
+
+ The `reason` argument indicates what caused the option to be implied.
+ It is necessary when it cannot be inferred from the `value`.
+ """
+
+ when = self._normalize_when(when, "imply_option")
+
+ # Don't do anything when --help was on the command line
+ if self._help:
+ return
+ if not reason and isinstance(value, SandboxDependsFunction):
+ deps = self._depends[value].dependencies
+ possible_reasons = [d for d in deps if d != self._help_option]
+ if len(possible_reasons) == 1:
+ if isinstance(possible_reasons[0], Option):
+ reason = possible_reasons[0]
+ if not reason and (
+ isinstance(value, (bool, tuple)) or isinstance(value, six.string_types)
+ ):
+ # A reason can be provided automatically when imply_option
+ # is called with an immediate value.
+ _, filename, line, _, _, _ = inspect.stack()[1]
+ reason = "imply_option at %s:%s" % (filename, line)
+
+ if not reason:
+ raise ConfigureError(
+ "Cannot infer what implies '%s'. Please add a `reason` to "
+ "the `imply_option` call." % option
+ )
+
+ prefix, name, values = Option.split_option(option)
+ if values != ():
+ raise ConfigureError("Implied option must not contain an '='")
+
+ self._implied_options.append(
+ ReadOnlyNamespace(
+ option=option,
+ prefix=prefix,
+ name=name,
+ value=value,
+ caller=inspect.stack()[1],
+ reason=reason,
+ when=when,
+ )
+ )
+
+ def _prepare_function(self, func, update_globals=None):
+ """Alter the given function global namespace with the common ground
+ for @depends, and @template.
+ """
+ if not inspect.isfunction(func):
+ raise TypeError("Unexpected type: '%s'" % type(func).__name__)
+ if func in self._prepared_functions:
+ return func
+
+ glob = SandboxedGlobal(
+ (k, v)
+ for k, v in six.iteritems(func.__globals__)
+ if (inspect.isfunction(v) and v not in self._templates)
+ or (inspect.isclass(v) and issubclass(v, Exception))
+ )
+ glob.update(
+ __builtins__=self.BUILTINS,
+ __file__=self._paths[-1] if self._paths else "",
+ __name__=self._paths[-1] if self._paths else "",
+ os=self.OS,
+ log=self.log_impl,
+ namespace=ReadOnlyNamespace,
+ )
+ if update_globals:
+ update_globals(glob)
+
+ # The execution model in the sandbox doesn't guarantee the execution
+ # order will always be the same for a given function, and if it uses
+ # variables from a closure that are changed after the function is
+ # declared, depending when the function is executed, the value of the
+ # variable can differ. For consistency, we force the function to use
+ # the value from the earliest it can be run, which is at declaration.
+ # Note this is not entirely bullet proof (if the value is e.g. a list,
+ # the list contents could have changed), but covers the bases.
+ closure = None
+ if func.__closure__:
+
+ def makecell(content):
+ def f():
+ content
+
+ return f.__closure__[0]
+
+ closure = tuple(makecell(cell.cell_contents) for cell in func.__closure__)
+
+ new_func = self.wraps(func)(
+ types.FunctionType(
+ func.__code__, glob, func.__name__, func.__defaults__, closure
+ )
+ )
+
+ @self.wraps(new_func)
+ def wrapped(*args, **kwargs):
+ if func in self._imports:
+ self._apply_imports(func, glob)
+ return new_func(*args, **kwargs)
+
+ self._prepared_functions.add(wrapped)
+ return wrapped
diff --git a/python/mozbuild/mozbuild/configure/check_debug_ranges.py b/python/mozbuild/mozbuild/configure/check_debug_ranges.py
new file mode 100644
index 0000000000..f82624c14f
--- /dev/null
+++ b/python/mozbuild/mozbuild/configure/check_debug_ranges.py
@@ -0,0 +1,68 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script returns the number of items for the DW_AT_ranges corresponding
+# to a given compilation unit. This is used as a helper to find a bug in some
+# versions of GNU ld.
+
+import re
+import subprocess
+import sys
+
+
+def get_range_for(compilation_unit, debug_info):
+ """Returns the range offset for a given compilation unit
+ in a given debug_info."""
+ name = ranges = ""
+ search_cu = False
+ for nfo in debug_info.splitlines():
+ if "DW_TAG_compile_unit" in nfo:
+ search_cu = True
+ elif "DW_TAG_" in nfo or not nfo.strip():
+ if name == compilation_unit and ranges != "":
+ return int(ranges, 16)
+ name = ranges = ""
+ search_cu = False
+ if search_cu:
+ if "DW_AT_name" in nfo:
+ name = nfo.rsplit(None, 1)[1]
+ elif "DW_AT_ranges" in nfo:
+ ranges = nfo.rsplit(None, 1)[1]
+ return None
+
+
+def get_range_length(range, debug_ranges):
+ """Returns the number of items in the range starting at the
+ given offset."""
+ length = 0
+ for line in debug_ranges.splitlines():
+ m = re.match("\s*([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([0-9a-fA-F]+)", line)
+ if m and int(m.group(1), 16) == range:
+ length += 1
+ return length
+
+
+def main(bin, compilation_unit):
+ p = subprocess.Popen(
+ ["objdump", "-W", bin],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ universal_newlines=True,
+ )
+ (out, err) = p.communicate()
+ sections = re.split("\n(Contents of the|The section) ", out)
+ debug_info = [s for s in sections if s.startswith(".debug_info")]
+ debug_ranges = [s for s in sections if s.startswith(".debug_ranges")]
+ if not debug_ranges or not debug_info:
+ return 0
+
+ range = get_range_for(compilation_unit, debug_info[0])
+ if range is not None:
+ return get_range_length(range, debug_ranges[0])
+
+ return -1
+
+
+if __name__ == "__main__":
+ print(main(*sys.argv[1:]))
diff --git a/python/mozbuild/mozbuild/configure/constants.py b/python/mozbuild/mozbuild/configure/constants.py
new file mode 100644
index 0000000000..a36152651d
--- /dev/null
+++ b/python/mozbuild/mozbuild/configure/constants.py
@@ -0,0 +1,131 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from collections import OrderedDict
+
+from mozbuild.util import EnumString
+
+CompilerType = EnumString.subclass(
+ "clang",
+ "clang-cl",
+ "gcc",
+ "msvc",
+)
+
+OS = EnumString.subclass(
+ "Android",
+ "DragonFly",
+ "FreeBSD",
+ "GNU",
+ "NetBSD",
+ "OpenBSD",
+ "OSX",
+ "SunOS",
+ "WINNT",
+ "WASI",
+)
+
+Kernel = EnumString.subclass(
+ "Darwin",
+ "DragonFly",
+ "FreeBSD",
+ "kFreeBSD",
+ "Linux",
+ "NetBSD",
+ "OpenBSD",
+ "SunOS",
+ "WINNT",
+ "WASI",
+)
+
+CPU_bitness = {
+ "aarch64": 64,
+ "Alpha": 64,
+ "arm": 32,
+ "hppa": 32,
+ "ia64": 64,
+ "loongarch64": 64,
+ "m68k": 32,
+ "mips32": 32,
+ "mips64": 64,
+ "ppc": 32,
+ "ppc64": 64,
+ "riscv64": 64,
+ "s390": 32,
+ "s390x": 64,
+ "sh4": 32,
+ "sparc": 32,
+ "sparc64": 64,
+ "x86": 32,
+ "x86_64": 64,
+ "wasm32": 32,
+}
+
+CPU = EnumString.subclass(*CPU_bitness.keys())
+
+Endianness = EnumString.subclass(
+ "big",
+ "little",
+)
+
+WindowsBinaryType = EnumString.subclass(
+ "win32",
+ "win64",
+)
+
+Abi = EnumString.subclass(
+ "msvc",
+ "mingw",
+)
+
+# The order of those checks matter
+CPU_preprocessor_checks = OrderedDict(
+ (
+ ("x86", "__i386__ || _M_IX86"),
+ ("x86_64", "__x86_64__ || _M_X64"),
+ ("arm", "__arm__ || _M_ARM"),
+ ("aarch64", "__aarch64__ || _M_ARM64"),
+ ("ia64", "__ia64__"),
+ ("s390x", "__s390x__"),
+ ("s390", "__s390__"),
+ ("ppc64", "__powerpc64__"),
+ ("ppc", "__powerpc__"),
+ ("Alpha", "__alpha__"),
+ ("hppa", "__hppa__"),
+ ("sparc64", "__sparc__ && __arch64__"),
+ ("sparc", "__sparc__"),
+ ("m68k", "__m68k__"),
+ ("mips64", "__mips64"),
+ ("mips32", "__mips__"),
+ ("riscv64", "__riscv && __riscv_xlen == 64"),
+ ("loongarch64", "__loongarch64"),
+ ("sh4", "__sh__"),
+ ("wasm32", "__wasm32__"),
+ )
+)
+
+assert sorted(CPU_preprocessor_checks.keys()) == sorted(CPU.POSSIBLE_VALUES)
+
+kernel_preprocessor_checks = {
+ "Darwin": "__APPLE__",
+ "DragonFly": "__DragonFly__",
+ "FreeBSD": "__FreeBSD__",
+ "kFreeBSD": "__FreeBSD_kernel__",
+ "Linux": "__linux__",
+ "NetBSD": "__NetBSD__",
+ "OpenBSD": "__OpenBSD__",
+ "SunOS": "__sun__",
+ "WINNT": "_WIN32 || __CYGWIN__",
+ "WASI": "__wasi__",
+}
+
+assert sorted(kernel_preprocessor_checks.keys()) == sorted(Kernel.POSSIBLE_VALUES)
+
+OS_preprocessor_checks = {
+ "Android": "__ANDROID__",
+}
+
+# We intentionally don't include all possible OSes in our checks, because we
+# only care about OS mismatches for specific target OSes.
+# assert sorted(OS_preprocessor_checks.keys()) == sorted(OS.POSSIBLE_VALUES)
diff --git a/python/mozbuild/mozbuild/configure/help.py b/python/mozbuild/mozbuild/configure/help.py
new file mode 100644
index 0000000000..bfd5e6ad6d
--- /dev/null
+++ b/python/mozbuild/mozbuild/configure/help.py
@@ -0,0 +1,90 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import re
+from collections import defaultdict
+
+from mozbuild.configure.options import Option
+
+
+class HelpFormatter(object):
+ def __init__(self, argv0):
+ self.intro = ["Usage: %s [options]" % os.path.basename(argv0)]
+ self.options = []
+
+ def add(self, option):
+ assert isinstance(option, Option)
+ if option.possible_origins == ("implied",):
+ # Don't display help if our option can only be implied.
+ return
+ self.options.append(option)
+
+ def format_options_by_category(self, options_by_category):
+ ret = []
+ for category, options in sorted(
+ options_by_category.items(), key=lambda x: x[0]
+ ):
+ ret.append(" " + category + ":")
+ for option in sorted(options, key=lambda opt: opt.option):
+ opt = option.option
+ if option.choices:
+ opt += "={%s}" % ",".join(option.choices)
+ help = self.format_help(option)
+ if len(option.default):
+ if help:
+ help += " "
+ help += "[%s]" % ",".join(option.default)
+
+ if len(opt) > 24 or not help:
+ ret.append(" %s" % opt)
+ if help:
+ ret.append("%s%s" % (" " * 30, help))
+ else:
+ ret.append(" %-24s %s" % (opt, help))
+ ret.append("")
+ return ret
+
+ RE_FORMAT = re.compile(r"{([^|}]*)\|([^|}]*)}")
+
+ # Return formatted help text for --{enable,disable,with,without}-* options.
+ #
+ # Format is the following syntax:
+ # {String for --enable or --with|String for --disable or --without}
+ #
+ # For example, '{Enable|Disable} optimizations' will be formatted to
+ # 'Enable optimizations' if the options's prefix is 'enable' or 'with',
+ # and formatted to 'Disable optimizations' if the options's prefix is
+ # 'disable' or 'without'.
+ def format_help(self, option):
+ if not option.help:
+ return ""
+
+ if option.prefix in ("enable", "with"):
+ replacement = r"\1"
+ elif option.prefix in ("disable", "without"):
+ replacement = r"\2"
+ else:
+ return option.help
+
+ return self.RE_FORMAT.sub(replacement, option.help)
+
+ def usage(self, out):
+ options_by_category = defaultdict(list)
+ env_by_category = defaultdict(list)
+ for option in self.options:
+ target = options_by_category if option.name else env_by_category
+ target[option.category].append(option)
+ options_formatted = [
+ "Options: [defaults in brackets after descriptions]"
+ ] + self.format_options_by_category(options_by_category)
+ env_formatted = ["Environment variables:"] + self.format_options_by_category(
+ env_by_category
+ )
+ print(
+ "\n\n".join(
+ "\n".join(t) for t in (self.intro, options_formatted, env_formatted)
+ ),
+ file=out,
+ )
diff --git a/python/mozbuild/mozbuild/configure/lint.py b/python/mozbuild/mozbuild/configure/lint.py
new file mode 100644
index 0000000000..7ea379b1ef
--- /dev/null
+++ b/python/mozbuild/mozbuild/configure/lint.py
@@ -0,0 +1,348 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import inspect
+import re
+import types
+from dis import Bytecode
+from functools import wraps
+from io import StringIO
+
+from mozbuild.util import memoize
+
+from . import (
+ CombinedDependsFunction,
+ ConfigureError,
+ ConfigureSandbox,
+ DependsFunction,
+ SandboxDependsFunction,
+ SandboxedGlobal,
+ TrivialDependsFunction,
+)
+from .help import HelpFormatter
+
+
+class LintSandbox(ConfigureSandbox):
+ def __init__(self, environ=None, argv=None, stdout=None, stderr=None):
+ out = StringIO()
+ stdout = stdout or out
+ stderr = stderr or out
+ environ = environ or {}
+ argv = argv or []
+ self._wrapped = {}
+ self._has_imports = set()
+ self._bool_options = []
+ self._bool_func_options = []
+ self.LOG = ""
+ super(LintSandbox, self).__init__(
+ {}, environ=environ, argv=argv, stdout=stdout, stderr=stderr
+ )
+
+ def run(self, path=None):
+ if path:
+ self.include_file(path)
+
+ for dep in self._depends.values():
+ self._check_dependencies(dep)
+
+ def _raise_from(self, exception, obj, line=0):
+ """
+ Raises the given exception as if it were emitted from the given
+ location.
+
+ The location is determined from the values of obj and line.
+ - `obj` can be a function or DependsFunction, in which case
+ `line` corresponds to the line within the function the exception
+ will be raised from (as an offset from the function's firstlineno).
+ - `obj` can be a stack frame, in which case `line` is ignored.
+ """
+
+ def thrower(e):
+ raise e
+
+ if isinstance(obj, DependsFunction):
+ obj, _ = self.unwrap(obj._func)
+
+ if inspect.isfunction(obj):
+ funcname = obj.__name__
+ filename = obj.__code__.co_filename
+ firstline = obj.__code__.co_firstlineno
+ line += firstline
+ elif inspect.isframe(obj):
+ funcname = obj.f_code.co_name
+ filename = obj.f_code.co_filename
+ firstline = obj.f_code.co_firstlineno
+ line = obj.f_lineno
+ else:
+ # Don't know how to handle the given location, still raise the
+ # exception.
+ raise exception
+
+ # Create a new function from the above thrower that pretends
+ # the `def` line is on the first line of the function given as
+ # argument, and the `raise` line is on the line given as argument.
+
+ offset = line - firstline
+ # co_lnotab is a string where each pair of consecutive character is
+ # (chr(byte_increment), chr(line_increment)), mapping bytes in co_code
+ # to line numbers relative to co_firstlineno.
+ # If the offset we need to encode is larger than what fits in a 8-bit
+ # signed integer, we need to split it.
+ co_lnotab = bytes([0, 127] * (offset // 127) + [0, offset % 127])
+ code = thrower.__code__
+ codetype_args = [
+ code.co_argcount,
+ code.co_kwonlyargcount,
+ code.co_nlocals,
+ code.co_stacksize,
+ code.co_flags,
+ code.co_code,
+ code.co_consts,
+ code.co_names,
+ code.co_varnames,
+ filename,
+ funcname,
+ firstline,
+ co_lnotab,
+ ]
+ if hasattr(code, "co_posonlyargcount"):
+ # co_posonlyargcount was introduced in Python 3.8.
+ codetype_args.insert(1, code.co_posonlyargcount)
+
+ code = types.CodeType(*codetype_args)
+ thrower = types.FunctionType(
+ code,
+ thrower.__globals__,
+ funcname,
+ thrower.__defaults__,
+ thrower.__closure__,
+ )
+ thrower(exception)
+
+ def _check_dependencies(self, obj):
+ if isinstance(obj, CombinedDependsFunction) or obj in (
+ self._always,
+ self._never,
+ ):
+ return
+ if not inspect.isroutine(obj._func):
+ return
+ func, glob = self.unwrap(obj._func)
+ func_args = inspect.getfullargspec(func)
+ if func_args.varkw:
+ e = ConfigureError(
+ "Keyword arguments are not allowed in @depends functions"
+ )
+ self._raise_from(e, func)
+
+ all_args = list(func_args.args)
+ if func_args.varargs:
+ all_args.append(func_args.varargs)
+ used_args = set()
+
+ for instr in Bytecode(func):
+ if instr.opname in ("LOAD_FAST", "LOAD_CLOSURE"):
+ if instr.argval in all_args:
+ used_args.add(instr.argval)
+
+ for num, arg in enumerate(all_args):
+ if arg not in used_args:
+ dep = obj.dependencies[num]
+ if dep != self._help_option or not self._need_help_dependency(obj):
+ if isinstance(dep, DependsFunction):
+ dep = dep.name
+ else:
+ dep = dep.option
+ e = ConfigureError("The dependency on `%s` is unused" % dep)
+ self._raise_from(e, func)
+
+ def _need_help_dependency(self, obj):
+ if isinstance(obj, (CombinedDependsFunction, TrivialDependsFunction)):
+ return False
+ if isinstance(obj, DependsFunction):
+ if obj in (self._always, self._never) or not inspect.isroutine(obj._func):
+ return False
+ func, glob = self.unwrap(obj._func)
+ # We allow missing --help dependencies for functions that:
+ # - don't use @imports
+ # - don't have a closure
+ # - don't use global variables
+ if func in self._has_imports or func.__closure__:
+ return True
+ for instr in Bytecode(func):
+ if instr.opname in ("LOAD_GLOBAL", "STORE_GLOBAL"):
+ # There is a fake os module when one is not imported,
+ # and it's allowed for functions without a --help
+ # dependency.
+ if instr.argval == "os" and glob.get("os") is self.OS:
+ continue
+ if instr.argval in self.BUILTINS:
+ continue
+ if instr.argval in "namespace":
+ continue
+ return True
+ return False
+
+ def _missing_help_dependency(self, obj):
+ if isinstance(obj, DependsFunction) and self._help_option in obj.dependencies:
+ return False
+ return self._need_help_dependency(obj)
+
+ @memoize
+ def _value_for_depends(self, obj):
+ with_help = self._help_option in obj.dependencies
+ if with_help:
+ for arg in obj.dependencies:
+ if self._missing_help_dependency(arg):
+ e = ConfigureError(
+ "Missing '--help' dependency because `%s` depends on "
+ "'--help' and `%s`" % (obj.name, arg.name)
+ )
+ self._raise_from(e, arg)
+ elif self._missing_help_dependency(obj):
+ e = ConfigureError("Missing '--help' dependency")
+ self._raise_from(e, obj)
+ return super(LintSandbox, self)._value_for_depends(obj)
+
+ def option_impl(self, *args, **kwargs):
+ result = super(LintSandbox, self).option_impl(*args, **kwargs)
+ when = self._conditions.get(result)
+ if when:
+ self._value_for(when)
+
+ self._check_option(result, *args, **kwargs)
+
+ return result
+
+ def _check_option(self, option, *args, **kwargs):
+ if "default" not in kwargs:
+ return
+ if len(args) == 0:
+ return
+
+ self._check_prefix_for_bool_option(*args, **kwargs)
+ self._check_help_for_option_with_func_default(option, *args, **kwargs)
+
+ def _check_prefix_for_bool_option(self, *args, **kwargs):
+ name = args[0]
+ default = kwargs["default"]
+
+ if type(default) != bool:
+ return
+
+ table = {
+ True: {
+ "enable": "disable",
+ "with": "without",
+ },
+ False: {
+ "disable": "enable",
+ "without": "with",
+ },
+ }
+ for prefix, replacement in table[default].items():
+ if name.startswith("--{}-".format(prefix)):
+ frame = inspect.currentframe()
+ while frame and frame.f_code.co_name != self.option_impl.__name__:
+ frame = frame.f_back
+ e = ConfigureError(
+ "{} should be used instead of "
+ "{} with default={}".format(
+ name.replace(
+ "--{}-".format(prefix), "--{}-".format(replacement)
+ ),
+ name,
+ default,
+ )
+ )
+ self._raise_from(e, frame.f_back if frame else None)
+
+ def _check_help_for_option_with_func_default(self, option, *args, **kwargs):
+ default = kwargs["default"]
+
+ if not isinstance(default, SandboxDependsFunction):
+ return
+
+ if not option.prefix:
+ return
+
+ default = self._resolve(default)
+ if type(default) is str:
+ return
+
+ help = kwargs["help"]
+ match = re.search(HelpFormatter.RE_FORMAT, help)
+ if match:
+ return
+
+ if option.prefix in ("enable", "disable"):
+ rule = "{Enable|Disable}"
+ else:
+ rule = "{With|Without}"
+
+ frame = inspect.currentframe()
+ while frame and frame.f_code.co_name != self.option_impl.__name__:
+ frame = frame.f_back
+ e = ConfigureError(
+ '`help` should contain "{}" because of non-constant default'.format(rule)
+ )
+ self._raise_from(e, frame.f_back if frame else None)
+
+ def unwrap(self, func):
+ glob = func.__globals__
+ while func in self._wrapped:
+ if isinstance(func.__globals__, SandboxedGlobal):
+ glob = func.__globals__
+ func = self._wrapped[func]
+ return func, glob
+
+ def wraps(self, func):
+ def do_wraps(wrapper):
+ self._wrapped[wrapper] = func
+ return wraps(func)(wrapper)
+
+ return do_wraps
+
+ def imports_impl(self, _import, _from=None, _as=None):
+ wrapper = super(LintSandbox, self).imports_impl(_import, _from=_from, _as=_as)
+
+ def decorator(func):
+ self._has_imports.add(func)
+ return wrapper(func)
+
+ return decorator
+
+ def _prepare_function(self, func, update_globals=None):
+ wrapped = super(LintSandbox, self)._prepare_function(func, update_globals)
+ _, glob = self.unwrap(wrapped)
+ imports = set()
+ for _from, _import, _as in self._imports.get(func, ()):
+ if _as:
+ imports.add(_as)
+ else:
+ what = _import.split(".")[0]
+ imports.add(what)
+ if _from == "__builtin__" and _import in glob["__builtins__"]:
+ e = NameError(
+ "builtin '{}' doesn't need to be imported".format(_import)
+ )
+ self._raise_from(e, func)
+ for instr in Bytecode(func):
+ code = func.__code__
+ if (
+ instr.opname == "LOAD_GLOBAL"
+ and instr.argval not in glob
+ and instr.argval not in imports
+ and instr.argval not in glob["__builtins__"]
+ and instr.argval not in code.co_varnames[: code.co_argcount]
+ ):
+ # Raise the same kind of error as what would happen during
+ # execution.
+ e = NameError("global name '{}' is not defined".format(instr.argval))
+ if instr.starts_line is None:
+ self._raise_from(e, func)
+ else:
+ self._raise_from(e, func, instr.starts_line - code.co_firstlineno)
+
+ return wrapped
diff --git a/python/mozbuild/mozbuild/configure/options.py b/python/mozbuild/mozbuild/configure/options.py
new file mode 100644
index 0000000000..cc3b4516ea
--- /dev/null
+++ b/python/mozbuild/mozbuild/configure/options.py
@@ -0,0 +1,614 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import inspect
+import os
+import sys
+from collections import OrderedDict
+
+import six
+
+HELP_OPTIONS_CATEGORY = "Help options"
+# List of whitelisted option categories. If you want to add a new category,
+# simply add it to this list; however, exercise discretion as
+# "./configure --help" becomes less useful if there are an excessive number of
+# categories.
+_ALL_CATEGORIES = (HELP_OPTIONS_CATEGORY,)
+
+
+def _infer_option_category(define_depth):
+ stack_frame = inspect.stack(0)[3 + define_depth]
+ try:
+ path = os.path.relpath(stack_frame[0].f_code.co_filename)
+ except ValueError:
+ # If this call fails, it means the relative path couldn't be determined
+ # (e.g. because this file is on a different drive than the cwd on a
+ # Windows machine). That's fine, just use the absolute filename.
+ path = stack_frame[0].f_code.co_filename
+ return "Options from " + path
+
+
+def istupleofstrings(obj):
+ return (
+ isinstance(obj, tuple)
+ and len(obj)
+ and all(isinstance(o, six.string_types) for o in obj)
+ )
+
+
+class OptionValue(tuple):
+ """Represents the value of a configure option.
+
+ This class is not meant to be used directly. Use its subclasses instead.
+
+ The `origin` attribute holds where the option comes from (e.g. environment,
+ command line, or default)
+ """
+
+ def __new__(cls, values=(), origin="unknown"):
+ return super(OptionValue, cls).__new__(cls, values)
+
+ def __init__(self, values=(), origin="unknown"):
+ self.origin = origin
+
+ def format(self, option):
+ if option.startswith("--"):
+ prefix, name, values = Option.split_option(option)
+ assert values == ()
+ for prefix_set in (
+ ("disable", "enable"),
+ ("without", "with"),
+ ):
+ if prefix in prefix_set:
+ prefix = prefix_set[int(bool(self))]
+ break
+ if prefix:
+ option = "--%s-%s" % (prefix, name)
+ elif self:
+ option = "--%s" % name
+ else:
+ return ""
+ if len(self):
+ return "%s=%s" % (option, ",".join(self))
+ return option
+ elif self and not len(self):
+ return "%s=1" % option
+ return "%s=%s" % (option, ",".join(self))
+
+ def __eq__(self, other):
+ # This is to catch naive comparisons against strings and other
+ # types in moz.configure files, as it is really easy to write
+ # value == 'foo'. We only raise a TypeError for instances that
+ # have content, because value-less instances (like PositiveOptionValue
+ # and NegativeOptionValue) are common and it is trivial to
+ # compare these.
+ if not isinstance(other, tuple) and len(self):
+ raise TypeError(
+ "cannot compare a populated %s against an %s; "
+ "OptionValue instances are tuples - did you mean to "
+ "compare against member elements using [x]?"
+ % (type(other).__name__, type(self).__name__)
+ )
+
+ # Allow explicit tuples to be compared.
+ if type(other) == tuple:
+ return tuple.__eq__(self, other)
+ elif isinstance(other, bool):
+ return bool(self) == other
+ # Else we're likely an OptionValue class.
+ elif type(other) != type(self):
+ return False
+ else:
+ return super(OptionValue, self).__eq__(other)
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __repr__(self):
+ return "%s%s" % (self.__class__.__name__, super(OptionValue, self).__repr__())
+
+ @staticmethod
+ def from_(value):
+ if isinstance(value, OptionValue):
+ return value
+ elif value is True:
+ return PositiveOptionValue()
+ elif value is False or value == ():
+ return NegativeOptionValue()
+ elif isinstance(value, six.string_types):
+ return PositiveOptionValue((value,))
+ elif isinstance(value, tuple):
+ return PositiveOptionValue(value)
+ else:
+ raise TypeError("Unexpected type: '%s'" % type(value).__name__)
+
+
+class PositiveOptionValue(OptionValue):
+ """Represents the value for a positive option (--enable/--with/--foo)
+ in the form of a tuple for when values are given to the option (in the form
+ --option=value[,value2...].
+ """
+
+ def __nonzero__(self): # py2
+ return True
+
+ def __bool__(self): # py3
+ return True
+
+
+class NegativeOptionValue(OptionValue):
+ """Represents the value for a negative option (--disable/--without)
+
+ This is effectively an empty tuple with a `origin` attribute.
+ """
+
+ def __new__(cls, origin="unknown"):
+ return super(NegativeOptionValue, cls).__new__(cls, origin=origin)
+
+ def __init__(self, origin="unknown"):
+ super(NegativeOptionValue, self).__init__(origin=origin)
+
+
+class InvalidOptionError(Exception):
+ pass
+
+
+class ConflictingOptionError(InvalidOptionError):
+ def __init__(self, message, **format_data):
+ if format_data:
+ message = message.format(**format_data)
+ super(ConflictingOptionError, self).__init__(message)
+ for k, v in six.iteritems(format_data):
+ setattr(self, k, v)
+
+
+class Option(object):
+ """Represents a configure option
+
+ A configure option can be a command line flag or an environment variable
+ or both.
+
+ - `name` is the full command line flag (e.g. --enable-foo).
+ - `env` is the environment variable name (e.g. ENV)
+ - `nargs` is the number of arguments the option may take. It can be a
+ number or the special values '?' (0 or 1), '*' (0 or more), or '+' (1 or
+ more).
+ - `default` can be used to give a default value to the option. When the
+ `name` of the option starts with '--enable-' or '--with-', the implied
+ default is an empty PositiveOptionValue. When it starts with '--disable-'
+ or '--without-', the implied default is a NegativeOptionValue.
+ - `choices` restricts the set of values that can be given to the option.
+ - `help` is the option description for use in the --help output.
+ - `possible_origins` is a tuple of strings that are origins accepted for
+ this option. Example origins are 'mozconfig', 'implied', and 'environment'.
+ - `category` is a human-readable string used only for categorizing command-
+ line options when displaying the output of `configure --help`. If not
+ supplied, the script will attempt to infer an appropriate category based
+ on the name of the file where the option was defined. If supplied it must
+ be in the _ALL_CATEGORIES list above.
+ - `define_depth` should generally only be used by templates that are used
+ to instantiate an option indirectly. Set this to a positive integer to
+ force the script to look into a deeper stack frame when inferring the
+ `category`.
+ """
+
+ __slots__ = (
+ "id",
+ "prefix",
+ "name",
+ "env",
+ "nargs",
+ "default",
+ "choices",
+ "help",
+ "possible_origins",
+ "category",
+ "define_depth",
+ )
+
+ def __init__(
+ self,
+ name=None,
+ env=None,
+ nargs=None,
+ default=None,
+ possible_origins=None,
+ choices=None,
+ category=None,
+ help=None,
+ define_depth=0,
+ ):
+ if not name and not env:
+ raise InvalidOptionError(
+ "At least an option name or an environment variable name must "
+ "be given"
+ )
+ if name:
+ if not isinstance(name, six.string_types):
+ raise InvalidOptionError("Option must be a string")
+ if not name.startswith("--"):
+ raise InvalidOptionError("Option must start with `--`")
+ if "=" in name:
+ raise InvalidOptionError("Option must not contain an `=`")
+ if not name.islower():
+ raise InvalidOptionError("Option must be all lowercase")
+ if env:
+ if not isinstance(env, six.string_types):
+ raise InvalidOptionError("Environment variable name must be a string")
+ if not env.isupper():
+ raise InvalidOptionError(
+ "Environment variable name must be all uppercase"
+ )
+ if nargs not in (None, "?", "*", "+") and not (
+ isinstance(nargs, int) and nargs >= 0
+ ):
+ raise InvalidOptionError(
+ "nargs must be a positive integer, '?', '*' or '+'"
+ )
+ if (
+ not isinstance(default, six.string_types)
+ and not isinstance(default, (bool, type(None)))
+ and not istupleofstrings(default)
+ ):
+ raise InvalidOptionError(
+ "default must be a bool, a string or a tuple of strings"
+ )
+ if choices and not istupleofstrings(choices):
+ raise InvalidOptionError("choices must be a tuple of strings")
+ if category and not isinstance(category, six.string_types):
+ raise InvalidOptionError("Category must be a string")
+ if category and category not in _ALL_CATEGORIES:
+ raise InvalidOptionError(
+ "Category must either be inferred or in the _ALL_CATEGORIES "
+ "list in options.py: %s" % ", ".join(_ALL_CATEGORIES)
+ )
+ if not isinstance(define_depth, int):
+ raise InvalidOptionError("DefineDepth must be an integer")
+ if not help:
+ raise InvalidOptionError("A help string must be provided")
+ if possible_origins and not istupleofstrings(possible_origins):
+ raise InvalidOptionError("possible_origins must be a tuple of strings")
+ self.possible_origins = possible_origins
+
+ if name:
+ prefix, name, values = self.split_option(name)
+ assert values == ()
+
+ # --disable and --without options mean the default is enabled.
+ # --enable and --with options mean the default is disabled.
+ # However, we allow a default to be given so that the default
+ # can be affected by other factors.
+ if prefix:
+ if default is None:
+ default = prefix in ("disable", "without")
+ elif default is False:
+ prefix = {
+ "disable": "enable",
+ "without": "with",
+ }.get(prefix, prefix)
+ elif default is True:
+ prefix = {
+ "enable": "disable",
+ "with": "without",
+ }.get(prefix, prefix)
+ else:
+ prefix = ""
+
+ self.prefix = prefix
+ self.name = name
+ self.env = env
+ if default in (None, False):
+ self.default = NegativeOptionValue(origin="default")
+ elif isinstance(default, tuple):
+ self.default = PositiveOptionValue(default, origin="default")
+ elif default is True:
+ self.default = PositiveOptionValue(origin="default")
+ else:
+ self.default = PositiveOptionValue((default,), origin="default")
+ if nargs is None:
+ nargs = 0
+ if len(self.default) == 1:
+ nargs = "?"
+ elif len(self.default) > 1:
+ nargs = "*"
+ elif choices:
+ nargs = 1
+ self.nargs = nargs
+ has_choices = choices is not None
+ if isinstance(self.default, PositiveOptionValue):
+ if has_choices and len(self.default) == 0:
+ raise InvalidOptionError(
+ "A `default` must be given along with `choices`"
+ )
+ if not self._validate_nargs(len(self.default)):
+ raise InvalidOptionError("The given `default` doesn't satisfy `nargs`")
+ if has_choices and not all(d in choices for d in self.default):
+ raise InvalidOptionError(
+ "The `default` value must be one of %s"
+ % ", ".join("'%s'" % c for c in choices)
+ )
+ elif has_choices:
+ maxargs = self.maxargs
+ if len(choices) < maxargs and maxargs != sys.maxsize:
+ raise InvalidOptionError("Not enough `choices` for `nargs`")
+ self.choices = choices
+ self.help = help
+ self.category = category or _infer_option_category(define_depth)
+
+ @staticmethod
+ def split_option(option):
+ """Split a flag or variable into a prefix, a name and values
+
+ Variables come in the form NAME=values (no prefix).
+ Flags come in the form --name=values or --prefix-name=values
+ where prefix is one of 'with', 'without', 'enable' or 'disable'.
+ The '=values' part is optional. Values are separated with commas.
+ """
+ if not isinstance(option, six.string_types):
+ raise InvalidOptionError("Option must be a string")
+
+ elements = option.split("=", 1)
+ name = elements[0]
+ values = tuple(elements[1].split(",")) if len(elements) == 2 else ()
+ if name.startswith("--"):
+ name = name[2:]
+ if not name.islower():
+ raise InvalidOptionError("Option must be all lowercase")
+ elements = name.split("-", 1)
+ prefix = elements[0]
+ if len(elements) == 2 and prefix in (
+ "enable",
+ "disable",
+ "with",
+ "without",
+ ):
+ return prefix, elements[1], values
+ else:
+ if name.startswith("-"):
+ raise InvalidOptionError(
+ "Option must start with two dashes instead of one"
+ )
+ if name.islower():
+ raise InvalidOptionError(
+ 'Environment variable name "%s" must be all uppercase' % name
+ )
+ return "", name, values
+
+ @staticmethod
+ def _join_option(prefix, name):
+ # The constraints around name and env in __init__ make it so that
+ # we can distinguish between flags and environment variables with
+ # islower/isupper.
+ if name.isupper():
+ assert not prefix
+ return name
+ elif prefix:
+ return "--%s-%s" % (prefix, name)
+ return "--%s" % name
+
+ @property
+ def option(self):
+ if self.prefix or self.name:
+ return self._join_option(self.prefix, self.name)
+ else:
+ return self.env
+
+ @property
+ def minargs(self):
+ if isinstance(self.nargs, int):
+ return self.nargs
+ return 1 if self.nargs == "+" else 0
+
+ @property
+ def maxargs(self):
+ if isinstance(self.nargs, int):
+ return self.nargs
+ return 1 if self.nargs == "?" else sys.maxsize
+
+ def _validate_nargs(self, num):
+ minargs, maxargs = self.minargs, self.maxargs
+ return num >= minargs and num <= maxargs
+
+ def get_value(self, option=None, origin="unknown"):
+ """Given a full command line option (e.g. --enable-foo=bar) or a
+ variable assignment (FOO=bar), returns the corresponding OptionValue.
+
+ Note: variable assignments can come from either the environment or
+ from the command line (e.g. `../configure CFLAGS=-O2`)
+ """
+ if not option:
+ return self.default
+
+ if self.possible_origins and origin not in self.possible_origins:
+ raise InvalidOptionError(
+ "%s can not be set by %s. Values are accepted from: %s"
+ % (option, origin, ", ".join(self.possible_origins))
+ )
+
+ prefix, name, values = self.split_option(option)
+ option = self._join_option(prefix, name)
+
+ assert name in (self.name, self.env)
+
+ if prefix in ("disable", "without"):
+ if values != ():
+ raise InvalidOptionError("Cannot pass a value to %s" % option)
+ return NegativeOptionValue(origin=origin)
+
+ if name == self.env:
+ if values == ("",):
+ return NegativeOptionValue(origin=origin)
+ if self.nargs in (0, "?", "*") and values == ("1",):
+ return PositiveOptionValue(origin=origin)
+
+ values = PositiveOptionValue(values, origin=origin)
+
+ if not self._validate_nargs(len(values)):
+ raise InvalidOptionError(
+ "%s takes %s value%s"
+ % (
+ option,
+ {
+ "?": "0 or 1",
+ "*": "0 or more",
+ "+": "1 or more",
+ }.get(self.nargs, str(self.nargs)),
+ "s" if (not isinstance(self.nargs, int) or self.nargs != 1) else "",
+ )
+ )
+
+ if len(values) and self.choices:
+ relative_result = None
+ for val in values:
+ if self.nargs in ("+", "*"):
+ if val.startswith(("+", "-")):
+ if relative_result is None:
+ relative_result = list(self.default)
+ sign = val[0]
+ val = val[1:]
+ if sign == "+":
+ if val not in relative_result:
+ relative_result.append(val)
+ else:
+ try:
+ relative_result.remove(val)
+ except ValueError:
+ pass
+
+ if val not in self.choices:
+ raise InvalidOptionError(
+ "'%s' is not one of %s"
+ % (val, ", ".join("'%s'" % c for c in self.choices))
+ )
+
+ if relative_result is not None:
+ values = PositiveOptionValue(relative_result, origin=origin)
+
+ return values
+
+ def __repr__(self):
+ return "<%s [%s]>" % (self.__class__.__name__, self.option)
+
+
+class CommandLineHelper(object):
+ """Helper class to handle the various ways options can be given either
+ on the command line of through the environment.
+
+ For instance, an Option('--foo', env='FOO') can be passed as --foo on the
+ command line, or as FOO=1 in the environment *or* on the command line.
+
+ If multiple variants are given, command line is prefered over the
+ environment, and if different values are given on the command line, the
+ last one wins. (This mimicks the behavior of autoconf, avoiding to break
+ existing mozconfigs using valid options in weird ways)
+
+ Extra options can be added afterwards through API calls. For those,
+ conflicting values will raise an exception.
+ """
+
+ def __init__(self, environ=os.environ, argv=sys.argv):
+ self._environ = dict(environ)
+ self._args = OrderedDict()
+ self._extra_args = OrderedDict()
+ self._origins = {}
+ self._last = 0
+
+ assert argv and not argv[0].startswith("--")
+ for arg in argv[1:]:
+ self.add(arg, "command-line", self._args)
+
+ def add(self, arg, origin="command-line", args=None):
+ assert origin != "default"
+ prefix, name, values = Option.split_option(arg)
+ if args is None:
+ args = self._extra_args
+ if args is self._extra_args and name in self._extra_args:
+ old_arg = self._extra_args[name][0]
+ old_prefix, _, old_values = Option.split_option(old_arg)
+ if prefix != old_prefix or values != old_values:
+ raise ConflictingOptionError(
+ "Cannot add '{arg}' to the {origin} set because it "
+ "conflicts with '{old_arg}' that was added earlier",
+ arg=arg,
+ origin=origin,
+ old_arg=old_arg,
+ old_origin=self._origins[old_arg],
+ )
+ self._last += 1
+ args[name] = arg, self._last
+ self._origins[arg] = origin
+
+ def _prepare(self, option, args):
+ arg = None
+ origin = "command-line"
+ from_name = args.get(option.name)
+ from_env = args.get(option.env)
+ if from_name and from_env:
+ arg1, pos1 = from_name
+ arg2, pos2 = from_env
+ arg, pos = (arg1, pos1) if abs(pos1) > abs(pos2) else (arg2, pos2)
+ if args is self._extra_args and (
+ option.get_value(arg1) != option.get_value(arg2)
+ ):
+ origin = self._origins[arg]
+ old_arg = arg2 if abs(pos1) > abs(pos2) else arg1
+ raise ConflictingOptionError(
+ "Cannot add '{arg}' to the {origin} set because it "
+ "conflicts with '{old_arg}' that was added earlier",
+ arg=arg,
+ origin=origin,
+ old_arg=old_arg,
+ old_origin=self._origins[old_arg],
+ )
+ elif from_name or from_env:
+ arg, pos = from_name if from_name else from_env
+ elif option.env and args is self._args:
+ env = self._environ.get(option.env)
+ if env is not None:
+ arg = "%s=%s" % (option.env, env)
+ origin = "environment"
+
+ origin = self._origins.get(arg, origin)
+
+ for k in (option.name, option.env):
+ try:
+ del args[k]
+ except KeyError:
+ pass
+
+ return arg, origin
+
+ def handle(self, option):
+ """Return the OptionValue corresponding to the given Option instance,
+ depending on the command line, environment, and extra arguments, and
+ the actual option or variable that set it.
+ Only works once for a given Option.
+ """
+ assert isinstance(option, Option)
+
+ arg, origin = self._prepare(option, self._args)
+ ret = option.get_value(arg, origin)
+
+ extra_arg, extra_origin = self._prepare(option, self._extra_args)
+ extra_ret = option.get_value(extra_arg, extra_origin)
+
+ if extra_ret.origin == "default":
+ return ret, arg
+
+ if ret.origin != "default" and extra_ret != ret:
+ raise ConflictingOptionError(
+ "Cannot add '{arg}' to the {origin} set because it conflicts "
+ "with {old_arg} from the {old_origin} set",
+ arg=extra_arg,
+ origin=extra_ret.origin,
+ old_arg=arg,
+ old_origin=ret.origin,
+ )
+
+ return extra_ret, extra_arg
+
+ def __iter__(self):
+ for d in (self._args, self._extra_args):
+ for arg, pos in six.itervalues(d):
+ yield arg
diff --git a/python/mozbuild/mozbuild/configure/util.py b/python/mozbuild/mozbuild/configure/util.py
new file mode 100644
index 0000000000..a58dc4d3f4
--- /dev/null
+++ b/python/mozbuild/mozbuild/configure/util.py
@@ -0,0 +1,235 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import codecs
+import io
+import itertools
+import locale
+import logging
+import os
+import sys
+from collections import deque
+from contextlib import contextmanager
+
+import six
+from looseversion import LooseVersion
+
+
+def getpreferredencoding():
+ # locale._parse_localename makes locale.getpreferredencoding
+ # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or
+ # 'ANSI_X3.4-1968' when it uses nl_langinfo.
+ encoding = None
+ try:
+ encoding = locale.getpreferredencoding()
+ except ValueError:
+ # On english OSX, LC_ALL is UTF-8 (not en-US.UTF-8), and
+ # that throws off locale._parse_localename, which ends up
+ # being used on e.g. homebrew python.
+ if os.environ.get("LC_ALL", "").upper() == "UTF-8":
+ encoding = "utf-8"
+ return encoding
+
+
+class Version(LooseVersion):
+ """A simple subclass of looseversion.LooseVersion.
+ Adds attributes for `major`, `minor`, `patch` for the first three
+ version components so users can easily pull out major/minor
+ versions, like:
+
+ v = Version('1.2b')
+ v.major == 1
+ v.minor == 2
+ v.patch == 0
+ """
+
+ def __init__(self, version):
+ # Can't use super, LooseVersion's base class is not a new-style class.
+ LooseVersion.__init__(self, version)
+ # Take the first three integer components, stopping at the first
+ # non-integer and padding the rest with zeroes.
+ (self.major, self.minor, self.patch) = list(
+ itertools.chain(
+ itertools.takewhile(lambda x: isinstance(x, int), self.version),
+ (0, 0, 0),
+ )
+ )[:3]
+
+
+class ConfigureOutputHandler(logging.Handler):
+ """A logging handler class that sends info messages to stdout and other
+ messages to stderr.
+
+ Messages sent to stdout are not formatted with the attached Formatter.
+ Additionally, if they end with '... ', no newline character is printed,
+ making the next message printed follow the '... '.
+
+ Only messages above log level INFO (included) are logged.
+
+ Messages below that level can be kept until an ERROR message is received,
+ at which point the last `maxlen` accumulated messages below INFO are
+ printed out. This feature is only enabled under the `queue_debug` context
+ manager.
+ """
+
+ def __init__(self, stdout=sys.stdout, stderr=sys.stderr, maxlen=20):
+ super(ConfigureOutputHandler, self).__init__()
+
+ # Python has this feature where it sets the encoding of pipes to
+ # ascii, which blatantly fails when trying to print out non-ascii.
+ def fix_encoding(fh):
+ if six.PY3:
+ return fh
+ try:
+ isatty = fh.isatty()
+ except AttributeError:
+ isatty = True
+
+ if not isatty:
+ encoding = getpreferredencoding()
+ if encoding:
+ return codecs.getwriter(encoding)(fh)
+ return fh
+
+ self._stdout = fix_encoding(stdout)
+ self._stderr = fix_encoding(stderr) if stdout != stderr else self._stdout
+ try:
+ fd1 = self._stdout.fileno()
+ fd2 = self._stderr.fileno()
+ self._same_output = self._is_same_output(fd1, fd2)
+ except (AttributeError, io.UnsupportedOperation):
+ self._same_output = self._stdout == self._stderr
+ self._stdout_waiting = None
+ self._debug = deque(maxlen=maxlen + 1)
+ self._keep_if_debug = self.THROW
+ self._queue_is_active = False
+
+ @staticmethod
+ def _is_same_output(fd1, fd2):
+ if fd1 == fd2:
+ return True
+ stat1 = os.fstat(fd1)
+ stat2 = os.fstat(fd2)
+ return stat1.st_ino == stat2.st_ino and stat1.st_dev == stat2.st_dev
+
+ # possible values for _stdout_waiting
+ WAITING = 1
+ INTERRUPTED = 2
+
+ # possible values for _keep_if_debug
+ THROW = 0
+ KEEP = 1
+ PRINT = 2
+
+ def emit(self, record):
+ try:
+ if record.levelno == logging.INFO:
+ stream = self._stdout
+ msg = six.ensure_text(record.getMessage())
+ if self._stdout_waiting == self.INTERRUPTED and self._same_output:
+ msg = " ... %s" % msg
+ self._stdout_waiting = msg.endswith("... ")
+ if msg.endswith("... "):
+ self._stdout_waiting = self.WAITING
+ else:
+ self._stdout_waiting = None
+ msg = "%s\n" % msg
+ elif record.levelno < logging.INFO and self._keep_if_debug != self.PRINT:
+ if self._keep_if_debug == self.KEEP:
+ self._debug.append(record)
+ return
+ else:
+ if record.levelno >= logging.ERROR and len(self._debug):
+ self._emit_queue()
+
+ if self._stdout_waiting == self.WAITING and self._same_output:
+ self._stdout_waiting = self.INTERRUPTED
+ self._stdout.write("\n")
+ self._stdout.flush()
+ stream = self._stderr
+ msg = "%s\n" % self.format(record)
+ stream.write(msg)
+ stream.flush()
+ except (KeyboardInterrupt, SystemExit, IOError):
+ raise
+ except Exception:
+ self.handleError(record)
+
+ @contextmanager
+ def queue_debug(self):
+ if self._queue_is_active:
+ yield
+ return
+ self._queue_is_active = True
+ self._keep_if_debug = self.KEEP
+ try:
+ yield
+ except Exception:
+ self._emit_queue()
+ # The exception will be handled and very probably printed out by
+ # something upper in the stack.
+ raise
+ finally:
+ self._queue_is_active = False
+ self._keep_if_debug = self.THROW
+ self._debug.clear()
+
+ def _emit_queue(self):
+ self._keep_if_debug = self.PRINT
+ if len(self._debug) == self._debug.maxlen:
+ r = self._debug.popleft()
+ self.emit(
+ logging.LogRecord(
+ r.name,
+ r.levelno,
+ r.pathname,
+ r.lineno,
+ "<truncated - see config.log for full output>",
+ (),
+ None,
+ )
+ )
+ while True:
+ try:
+ self.emit(self._debug.popleft())
+ except IndexError:
+ break
+ self._keep_if_debug = self.KEEP
+
+
+class LineIO(object):
+ """File-like class that sends each line of the written data to a callback
+ (without carriage returns).
+ """
+
+ def __init__(self, callback, errors="strict"):
+ self._callback = callback
+ self._buf = ""
+ self._encoding = getpreferredencoding()
+ self._errors = errors
+
+ def write(self, buf):
+ buf = six.ensure_text(buf, encoding=self._encoding or "utf-8")
+ lines = buf.splitlines()
+ if not lines:
+ return
+ if self._buf:
+ lines[0] = self._buf + lines[0]
+ self._buf = ""
+ if not buf.endswith("\n"):
+ self._buf = lines.pop()
+
+ for line in lines:
+ self._callback(line)
+
+ def close(self):
+ if self._buf:
+ self._callback(self._buf)
+ self._buf = ""
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-06 10:39:52 +0000