diff options
Diffstat (limited to 'python/mozbuild/mozbuild/backend/configenvironment.py')
| -rw-r--r-- | python/mozbuild/mozbuild/backend/configenvironment.py | 357 |
1 files changed, 357 insertions, 0 deletions
diff --git a/python/mozbuild/mozbuild/backend/configenvironment.py b/python/mozbuild/mozbuild/backend/configenvironment.py new file mode 100644 index 0000000000..eef1b62ee6 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/configenvironment.py @@ -0,0 +1,357 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import sys +from collections import OrderedDict +from collections.abc import Iterable +from pathlib import Path +from types import ModuleType + +import mozpack.path as mozpath +import six + +from mozbuild.shellutil import quote as shell_quote +from mozbuild.util import ( + FileAvoidWrite, + ReadOnlyDict, + memoized_property, + system_encoding, +) + + +class ConfigStatusFailure(Exception): + """Error loading config.status""" + + +class BuildConfig(object): + """Represents the output of configure.""" + + _CODE_CACHE = {} + + def __init__(self): + self.topsrcdir = None + self.topobjdir = None + self.defines = {} + self.substs = {} + self.files = [] + self.mozconfig = None + + @classmethod + def from_config_status(cls, path): + """Create an instance from a config.status file.""" + code_cache = cls._CODE_CACHE + mtime = os.path.getmtime(path) + + # cache the compiled code as it can be reused + # we cache it the first time, or if the file changed + if path not in code_cache or code_cache[path][0] != mtime: + # Add config.status manually to sys.modules so it gets picked up by + # iter_modules_in_path() for automatic dependencies. + mod = ModuleType("config.status") + mod.__file__ = path + sys.modules["config.status"] = mod + + with open(path, "rt") as fh: + source = fh.read() + code_cache[path] = ( + mtime, + compile(source, path, "exec", dont_inherit=1), + ) + + g = {"__builtins__": __builtins__, "__file__": path} + l = {} + try: + exec(code_cache[path][1], g, l) + except Exception: + raise ConfigStatusFailure() + + config = BuildConfig() + + for name in l["__all__"]: + setattr(config, name, l[name]) + + return config + + +class ConfigEnvironment(object): + """Perform actions associated with a configured but bare objdir. + + The purpose of this class is to preprocess files from the source directory + and output results in the object directory. + + There are two types of files: config files and config headers, + each treated through a different member function. + + Creating a ConfigEnvironment requires a few arguments: + - topsrcdir and topobjdir are, respectively, the top source and + the top object directory. + - defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in autoconf. + - substs is a dict filled from AC_SUBST in autoconf. + + ConfigEnvironment automatically defines one additional substs variable + from all the defines: + - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on + preprocessor command lines. The order in which defines were given + when creating the ConfigEnvironment is preserved. + + and two other additional subst variables from all the other substs: + - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted + order, for use in autoconf.mk. It includes ACDEFINES. + Only substs with a VALUE are included, such that the resulting file + doesn't change when new empty substs are added. + This results in less invalidation of build dependencies in the case + of autoconf.mk.. + - ALLEMPTYSUBSTS contains the substs with an empty value, in the form NAME =. + + ConfigEnvironment expects a "top_srcdir" subst to be set with the top + source directory, in msys format on windows. It is used to derive a + "srcdir" subst when treating config files. It can either be an absolute + path or a path relative to the topobjdir. + """ + + def __init__( + self, + topsrcdir, + topobjdir, + defines=None, + substs=None, + source=None, + mozconfig=None, + ): + + if not source: + source = mozpath.join(topobjdir, "config.status") + self.source = source + self.defines = ReadOnlyDict(defines or {}) + self.substs = dict(substs or {}) + self.topsrcdir = mozpath.abspath(topsrcdir) + self.topobjdir = mozpath.abspath(topobjdir) + self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None + self.lib_prefix = self.substs.get("LIB_PREFIX", "") + if "LIB_SUFFIX" in self.substs: + self.lib_suffix = ".%s" % self.substs["LIB_SUFFIX"] + self.dll_prefix = self.substs.get("DLL_PREFIX", "") + self.dll_suffix = self.substs.get("DLL_SUFFIX", "") + self.host_dll_prefix = self.substs.get("HOST_DLL_PREFIX", "") + self.host_dll_suffix = self.substs.get("HOST_DLL_SUFFIX", "") + if self.substs.get("IMPORT_LIB_SUFFIX"): + self.import_prefix = self.lib_prefix + self.import_suffix = ".%s" % self.substs["IMPORT_LIB_SUFFIX"] + else: + self.import_prefix = self.dll_prefix + self.import_suffix = self.dll_suffix + if self.substs.get("HOST_IMPORT_LIB_SUFFIX"): + self.host_import_prefix = self.substs.get("HOST_LIB_PREFIX", "") + self.host_import_suffix = ".%s" % self.substs["HOST_IMPORT_LIB_SUFFIX"] + else: + self.host_import_prefix = self.host_dll_prefix + self.host_import_suffix = self.host_dll_suffix + self.bin_suffix = self.substs.get("BIN_SUFFIX", "") + + global_defines = [name for name in self.defines] + self.substs["ACDEFINES"] = " ".join( + [ + "-D%s=%s" % (name, shell_quote(self.defines[name]).replace("$", "$$")) + for name in sorted(global_defines) + ] + ) + + def serialize(name, obj): + if isinstance(obj, six.string_types): + return obj + if isinstance(obj, Iterable): + return " ".join(obj) + raise Exception("Unhandled type %s for %s", type(obj), str(name)) + + self.substs["ALLSUBSTS"] = "\n".join( + sorted( + [ + "%s = %s" % (name, serialize(name, self.substs[name])) + for name in self.substs + if self.substs[name] + ] + ) + ) + self.substs["ALLEMPTYSUBSTS"] = "\n".join( + sorted(["%s =" % name for name in self.substs if not self.substs[name]]) + ) + + self.substs = ReadOnlyDict(self.substs) + + @property + def is_artifact_build(self): + return self.substs.get("MOZ_ARTIFACT_BUILDS", False) + + @memoized_property + def acdefines(self): + acdefines = dict((name, self.defines[name]) for name in self.defines) + return ReadOnlyDict(acdefines) + + @staticmethod + def from_config_status(path): + config = BuildConfig.from_config_status(path) + + return ConfigEnvironment( + config.topsrcdir, config.topobjdir, config.defines, config.substs, path + ) + + +class PartialConfigDict(object): + """Facilitates mapping the config.statusd defines & substs with dict-like access. + + This allows a buildconfig client to use buildconfig.defines['FOO'] (and + similar for substs), where the value of FOO is delay-loaded until it is + needed. + """ + + def __init__(self, config_statusd, typ, environ_override=False): + self._dict = {} + self._datadir = mozpath.join(config_statusd, typ) + self._config_track = mozpath.join(self._datadir, "config.track") + self._files = set() + self._environ_override = environ_override + + def _load_config_track(self): + existing_files = set() + try: + with open(self._config_track) as fh: + existing_files.update(fh.read().splitlines()) + except IOError: + pass + return existing_files + + def _write_file(self, key, value): + filename = mozpath.join(self._datadir, key) + with FileAvoidWrite(filename) as fh: + to_write = json.dumps(value, indent=4) + fh.write(to_write.encode(system_encoding)) + return filename + + def _fill_group(self, values): + # Clear out any cached values. This is mostly for tests that will check + # the environment, write out a new set of variables, and then check the + # environment again. Normally only configure ends up calling this + # function, and other consumers create their own + # PartialConfigEnvironments in new python processes. + self._dict = {} + + existing_files = self._load_config_track() + existing_files = {Path(f) for f in existing_files} + + new_files = set() + for k, v in six.iteritems(values): + new_files.add(Path(self._write_file(k, v))) + + for filename in existing_files - new_files: + # We can't actually os.remove() here, since make would not see that the + # file has been removed and that the target needs to be updated. Instead + # we just overwrite the file with a value of None, which is equivalent + # to a non-existing file. + with FileAvoidWrite(filename) as fh: + json.dump(None, fh) + + with FileAvoidWrite(self._config_track) as fh: + for f in sorted(new_files): + fh.write("%s\n" % f) + + def __getitem__(self, key): + if self._environ_override: + if (key not in ("CPP", "CXXCPP", "SHELL")) and (key in os.environ): + return os.environ[key] + + if key not in self._dict: + data = None + try: + filename = mozpath.join(self._datadir, key) + self._files.add(filename) + with open(filename) as f: + data = json.load(f) + except IOError: + pass + self._dict[key] = data + + if self._dict[key] is None: + raise KeyError("'%s'" % key) + return self._dict[key] + + def __setitem__(self, key, value): + self._dict[key] = value + + def get(self, key, default=None): + return self[key] if key in self else default + + def __contains__(self, key): + try: + return self[key] is not None + except KeyError: + return False + + def iteritems(self): + existing_files = self._load_config_track() + for f in existing_files: + # The track file contains filenames, and the basename is the + # variable name. + var = mozpath.basename(f) + yield var, self[var] + + +class PartialConfigEnvironment(object): + """Allows access to individual config.status items via config.statusd/* files. + + This class is similar to the full ConfigEnvironment, which uses + config.status, except this allows access and tracks dependencies to + individual configure values. It is intended to be used during the build + process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and + anything else that may need to access specific substs or defines. + + Creating a PartialConfigEnvironment requires only the topobjdir, which is + needed to distinguish between the top-level environment and the js/src + environment. + + The PartialConfigEnvironment automatically defines one additional subst variable + from all the defines: + + - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on + preprocessor command lines. The order in which defines were given + when creating the ConfigEnvironment is preserved. + + and one additional define from all the defines as a dictionary: + + - ALLDEFINES contains all of the global defines as a dictionary. This is + intended to be used instead of the defines structure from config.status so + that scripts can depend directly on its value. + """ + + def __init__(self, topobjdir): + config_statusd = mozpath.join(topobjdir, "config.statusd") + self.substs = PartialConfigDict(config_statusd, "substs", environ_override=True) + self.defines = PartialConfigDict(config_statusd, "defines") + self.topobjdir = topobjdir + + def write_vars(self, config): + substs = config["substs"].copy() + defines = config["defines"].copy() + + global_defines = [name for name in config["defines"]] + acdefines = " ".join( + [ + "-D%s=%s" + % (name, shell_quote(config["defines"][name]).replace("$", "$$")) + for name in sorted(global_defines) + ] + ) + substs["ACDEFINES"] = acdefines + + all_defines = OrderedDict() + for k in global_defines: + all_defines[k] = config["defines"][k] + defines["ALLDEFINES"] = all_defines + + self.substs._fill_group(substs) + self.defines._fill_group(defines) + + def get_dependencies(self): + return ["$(wildcard %s)" % f for f in self.substs._files | self.defines._files] |