diff options
Diffstat (limited to 'python/mozbuild/mozbuild/frontend/data.py')
| -rw-r--r-- | python/mozbuild/mozbuild/frontend/data.py | 1369 |
1 files changed, 1369 insertions, 0 deletions
diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py new file mode 100644 index 0000000000..84a47f90cf --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -0,0 +1,1369 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +r"""Data structures representing Mozilla's source tree. + +The frontend files are parsed into static data structures. These data +structures are defined in this module. + +All data structures of interest are children of the TreeMetadata class. + +Logic for populating these data structures is not defined in this class. +Instead, what we have here are dumb container classes. The emitter module +contains the code for converting executed mozbuild files into these data +structures. +""" + +from collections import OrderedDict, defaultdict + +import mozpack.path as mozpath +import six +from mozpack.chrome.manifest import ManifestEntry + +from mozbuild.frontend.context import ObjDirPath, SourcePath + +from ..testing import all_test_flavors +from ..util import group_unified_files +from .context import FinalTargetValue + + +class TreeMetadata(object): + """Base class for all data being captured.""" + + __slots__ = () + + def to_dict(self): + return {k.lower(): getattr(self, k) for k in self.DICT_ATTRS} + + +class ContextDerived(TreeMetadata): + """Build object derived from a single Context instance. + + It holds fields common to all context derived classes. This class is likely + never instantiated directly but is instead derived from. + """ + + __slots__ = ( + "context_main_path", + "context_all_paths", + "topsrcdir", + "topobjdir", + "relsrcdir", + "srcdir", + "objdir", + "config", + "_context", + ) + + def __init__(self, context): + TreeMetadata.__init__(self) + + # Capture the files that were evaluated to fill this context. + self.context_main_path = context.main_path + self.context_all_paths = context.all_paths + + # Basic directory state. + self.topsrcdir = context.config.topsrcdir + self.topobjdir = context.config.topobjdir + + self.relsrcdir = context.relsrcdir + self.srcdir = context.srcdir + self.objdir = context.objdir + + self.config = context.config + + self._context = context + + @property + def install_target(self): + return self._context["FINAL_TARGET"] + + @property + def installed(self): + return self._context["DIST_INSTALL"] is not False + + @property + def defines(self): + defines = self._context["DEFINES"] + return Defines(self._context, defines) if defines else None + + @property + def relobjdir(self): + return mozpath.relpath(self.objdir, self.topobjdir) + + +class HostMixin(object): + @property + def defines(self): + defines = self._context["HOST_DEFINES"] + return HostDefines(self._context, defines) if defines else None + + +class DirectoryTraversal(ContextDerived): + """Describes how directory traversal for building should work. + + This build object is likely only of interest to the recursive make backend. + Other build backends should (ideally) not attempt to mimic the behavior of + the recursive make backend. The only reason this exists is to support the + existing recursive make backend while the transition to mozbuild frontend + files is complete and we move to a more optimal build backend. + + Fields in this class correspond to similarly named variables in the + frontend files. + """ + + __slots__ = ("dirs",) + + def __init__(self, context): + ContextDerived.__init__(self, context) + + self.dirs = [] + + +class BaseConfigSubstitution(ContextDerived): + """Base class describing autogenerated files as part of config.status.""" + + __slots__ = ("input_path", "output_path", "relpath") + + def __init__(self, context): + ContextDerived.__init__(self, context) + + self.input_path = None + self.output_path = None + self.relpath = None + + +class ConfigFileSubstitution(BaseConfigSubstitution): + """Describes a config file that will be generated using substitutions.""" + + +class VariablePassthru(ContextDerived): + """A dict of variables to pass through to backend.mk unaltered. + + The purpose of this object is to facilitate rapid transitioning of + variables from Makefile.in to moz.build. In the ideal world, this class + does not exist and every variable has a richer class representing it. + As long as we rely on this class, we lose the ability to have flexibility + in our build backends since we will continue to be tied to our rules.mk. + """ + + __slots__ = "variables" + + def __init__(self, context): + ContextDerived.__init__(self, context) + self.variables = {} + + +class ComputedFlags(ContextDerived): + """Aggregate flags for consumption by various backends.""" + + __slots__ = ("flags",) + + def __init__(self, context, reader_flags): + ContextDerived.__init__(self, context) + self.flags = reader_flags + + def resolve_flags(self, key, value): + # Bypass checks done by CompileFlags that would keep us from + # setting a value here. + dict.__setitem__(self.flags, key, value) + + def get_flags(self): + flags = defaultdict(list) + for key, _, dest_vars in self.flags.flag_variables: + value = self.flags.get(key) + if value: + for dest_var in dest_vars: + flags[dest_var].extend(value) + return sorted(flags.items()) + + +class XPIDLModule(ContextDerived): + """Describes an XPIDL module to be compiled.""" + + __slots__ = ("name", "idl_files") + + def __init__(self, context, name, idl_files): + ContextDerived.__init__(self, context) + + assert all(isinstance(idl, SourcePath) for idl in idl_files) + self.name = name + self.idl_files = idl_files + + +class BaseDefines(ContextDerived): + """Context derived container object for DEFINES/HOST_DEFINES, + which are OrderedDicts. + """ + + __slots__ = "defines" + + def __init__(self, context, defines): + ContextDerived.__init__(self, context) + self.defines = defines + + def get_defines(self): + for define, value in six.iteritems(self.defines): + if value is True: + yield ("-D%s" % define) + elif value is False: + yield ("-U%s" % define) + else: + yield ("-D%s=%s" % (define, value)) + + def update(self, more_defines): + if isinstance(more_defines, Defines): + self.defines.update(more_defines.defines) + else: + self.defines.update(more_defines) + + +class Defines(BaseDefines): + pass + + +class HostDefines(BaseDefines): + pass + + +class WasmDefines(BaseDefines): + pass + + +class WebIDLCollection(ContextDerived): + """Collects WebIDL info referenced during the build.""" + + def __init__(self, context): + ContextDerived.__init__(self, context) + self.sources = set() + self.generated_sources = set() + self.generated_events_sources = set() + self.preprocessed_sources = set() + self.test_sources = set() + self.preprocessed_test_sources = set() + self.example_interfaces = set() + + def all_regular_sources(self): + return ( + self.sources + | self.generated_sources + | self.generated_events_sources + | self.preprocessed_sources + ) + + def all_regular_basenames(self): + return [mozpath.basename(source) for source in self.all_regular_sources()] + + def all_regular_stems(self): + return [mozpath.splitext(b)[0] for b in self.all_regular_basenames()] + + def all_regular_bindinggen_stems(self): + for stem in self.all_regular_stems(): + yield "%sBinding" % stem + + for source in self.generated_events_sources: + yield mozpath.splitext(mozpath.basename(source))[0] + + def all_regular_cpp_basenames(self): + for stem in self.all_regular_bindinggen_stems(): + yield "%s.cpp" % stem + + def all_test_sources(self): + return self.test_sources | self.preprocessed_test_sources + + def all_test_basenames(self): + return [mozpath.basename(source) for source in self.all_test_sources()] + + def all_test_stems(self): + return [mozpath.splitext(b)[0] for b in self.all_test_basenames()] + + def all_test_cpp_basenames(self): + return sorted("%sBinding.cpp" % s for s in self.all_test_stems()) + + def all_static_sources(self): + return self.sources | self.generated_events_sources | self.test_sources + + def all_non_static_sources(self): + return self.generated_sources | self.all_preprocessed_sources() + + def all_non_static_basenames(self): + return [mozpath.basename(s) for s in self.all_non_static_sources()] + + def all_preprocessed_sources(self): + return self.preprocessed_sources | self.preprocessed_test_sources + + def all_sources(self): + return set(self.all_regular_sources()) | set(self.all_test_sources()) + + def all_basenames(self): + return [mozpath.basename(source) for source in self.all_sources()] + + def all_stems(self): + return [mozpath.splitext(b)[0] for b in self.all_basenames()] + + def generated_events_basenames(self): + return [mozpath.basename(s) for s in self.generated_events_sources] + + def generated_events_stems(self): + return [mozpath.splitext(b)[0] for b in self.generated_events_basenames()] + + @property + def unified_source_mapping(self): + # Bindings are compiled in unified mode to speed up compilation and + # to reduce linker memory size. Note that test bindings are separated + # from regular ones so tests bindings aren't shipped. + return list( + group_unified_files( + sorted(self.all_regular_cpp_basenames()), + unified_prefix="UnifiedBindings", + unified_suffix="cpp", + files_per_unified_file=32, + ) + ) + + def all_source_files(self): + from mozwebidlcodegen import WebIDLCodegenManager + + return sorted(list(WebIDLCodegenManager.GLOBAL_DEFINE_FILES)) + sorted( + set(p for p, _ in self.unified_source_mapping) + ) + + +class IPDLCollection(ContextDerived): + """Collects IPDL files during the build.""" + + def __init__(self, context): + ContextDerived.__init__(self, context) + self.sources = set() + self.preprocessed_sources = set() + + def all_sources(self): + return self.sources | self.preprocessed_sources + + def all_regular_sources(self): + return self.sources + + def all_preprocessed_sources(self): + return self.preprocessed_sources + + def all_source_files(self): + # Source files generated by IPDL are built as generated UnifiedSources + # from the context which included the IPDL file, rather than the context + # which builds the IPDLCollection, so we report no files here. + return [] + + +class XPCOMComponentManifests(ContextDerived): + """Collects XPCOM manifest files during the build.""" + + def __init__(self, context): + ContextDerived.__init__(self, context) + self.manifests = set() + + def all_sources(self): + return self.manifests + + def all_source_files(self): + return [] + + +class LinkageWrongKindError(Exception): + """Error thrown when trying to link objects of the wrong kind""" + + +class Linkable(ContextDerived): + """Generic context derived container object for programs and libraries""" + + __slots__ = ( + "cxx_link", + "lib_defines", + "linked_libraries", + "linked_system_libs", + "sources", + ) + + def __init__(self, context): + ContextDerived.__init__(self, context) + self.cxx_link = False + self.linked_libraries = [] + self.linked_system_libs = [] + self.lib_defines = Defines(context, OrderedDict()) + self.sources = defaultdict(list) + + def link_library(self, obj): + assert isinstance(obj, BaseLibrary) + if obj.KIND != self.KIND: + raise LinkageWrongKindError("%s != %s" % (obj.KIND, self.KIND)) + self.linked_libraries.append(obj) + if obj.cxx_link and not isinstance(obj, SharedLibrary): + self.cxx_link = True + obj.refs.append(self) + + def link_system_library(self, lib): + # The '$' check is here as a special temporary rule, allowing the + # inherited use of make variables, most notably in TK_LIBS. + if not lib.startswith("$") and not lib.startswith("-"): + type_var = "HOST_CC_TYPE" if self.KIND == "host" else "CC_TYPE" + compiler_type = self.config.substs.get(type_var) + if compiler_type in ("gcc", "clang"): + lib = "-l%s" % lib + elif self.KIND == "host": + lib = "%s%s%s" % ( + self.config.host_import_prefix, + lib, + self.config.host_import_suffix, + ) + else: + lib = "%s%s%s" % ( + self.config.import_prefix, + lib, + self.config.import_suffix, + ) + self.linked_system_libs.append(lib) + + def source_files(self): + all_sources = [] + # This is ordered for reproducibility and consistently w/ + # config/rules.mk + for suffix in (".c", ".S", ".cpp", ".m", ".mm", ".s"): + all_sources += self.sources.get(suffix, []) + return all_sources + + def _get_objs(self, sources): + obj_prefix = "" + if self.KIND == "host": + obj_prefix = "host_" + + return [ + mozpath.join( + self.objdir, + "%s%s.%s" + % ( + obj_prefix, + mozpath.splitext(mozpath.basename(f))[0], + self._obj_suffix(), + ), + ) + for f in sources + ] + + def _obj_suffix(self): + """Can be overridden by a base class for custom behavior.""" + return self.config.substs.get("OBJ_SUFFIX", "") + + @property + def objs(self): + return self._get_objs(self.source_files()) + + +class BaseProgram(Linkable): + """Context derived container object for programs, which is a unicode + string. + + This class handles automatically appending a binary suffix to the program + name. + If the suffix is not defined, the program name is unchanged. + Otherwise, if the program name ends with the given suffix, it is unchanged + Otherwise, the suffix is appended to the program name. + """ + + __slots__ = "program" + + DICT_ATTRS = {"install_target", "KIND", "program", "relobjdir"} + + def __init__(self, context, program, is_unit_test=False): + Linkable.__init__(self, context) + + bin_suffix = context.config.substs.get(self.SUFFIX_VAR, "") + if not program.endswith(bin_suffix): + program += bin_suffix + self.program = program + self.is_unit_test = is_unit_test + + @property + def output_path(self): + if self.installed: + return ObjDirPath( + self._context, "!/" + mozpath.join(self.install_target, self.program) + ) + else: + return ObjDirPath(self._context, "!" + self.program) + + def __repr__(self): + return "<%s: %s/%s>" % (type(self).__name__, self.relobjdir, self.program) + + @property + def name(self): + return self.program + + +class Program(BaseProgram): + """Context derived container object for PROGRAM""" + + SUFFIX_VAR = "BIN_SUFFIX" + KIND = "target" + + +class HostProgram(HostMixin, BaseProgram): + """Context derived container object for HOST_PROGRAM""" + + SUFFIX_VAR = "HOST_BIN_SUFFIX" + KIND = "host" + + @property + def install_target(self): + return "dist/host/bin" + + +class SimpleProgram(BaseProgram): + """Context derived container object for each program in SIMPLE_PROGRAMS""" + + SUFFIX_VAR = "BIN_SUFFIX" + KIND = "target" + + def source_files(self): + for srcs in self.sources.values(): + for f in srcs: + if ( + mozpath.basename(mozpath.splitext(f)[0]) + == mozpath.splitext(self.program)[0] + ): + return [f] + return [] + + +class HostSimpleProgram(HostMixin, BaseProgram): + """Context derived container object for each program in + HOST_SIMPLE_PROGRAMS""" + + SUFFIX_VAR = "HOST_BIN_SUFFIX" + KIND = "host" + + def source_files(self): + for srcs in self.sources.values(): + for f in srcs: + if ( + "host_%s" % mozpath.basename(mozpath.splitext(f)[0]) + == mozpath.splitext(self.program)[0] + ): + return [f] + return [] + + +def cargo_output_directory(context, target_var): + # cargo creates several directories and places its build artifacts + # in those directories. The directory structure depends not only + # on the target, but also what sort of build we are doing. + rust_build_kind = "release" + if context.config.substs.get("MOZ_DEBUG_RUST"): + rust_build_kind = "debug" + return mozpath.join(context.config.substs[target_var], rust_build_kind) + + +# Rust programs aren't really Linkable, since Cargo handles all the details +# of linking things. +class BaseRustProgram(ContextDerived): + __slots__ = ( + "name", + "cargo_file", + "location", + "SUFFIX_VAR", + "KIND", + "TARGET_SUBST_VAR", + ) + + def __init__(self, context, name, cargo_file): + ContextDerived.__init__(self, context) + self.name = name + self.cargo_file = cargo_file + # Skip setting properties below which depend on cargo + # when we don't have a compile environment. The required + # config keys won't be available, but the instance variables + # that we don't set should never be accessed by the actual + # build in that case. + if not context.config.substs.get("COMPILE_ENVIRONMENT"): + return + cargo_dir = cargo_output_directory(context, self.TARGET_SUBST_VAR) + exe_file = "%s%s" % (name, context.config.substs.get(self.SUFFIX_VAR, "")) + self.location = mozpath.join(cargo_dir, exe_file) + + +class RustProgram(BaseRustProgram): + SUFFIX_VAR = "BIN_SUFFIX" + KIND = "target" + TARGET_SUBST_VAR = "RUST_TARGET" + + +class HostRustProgram(BaseRustProgram): + SUFFIX_VAR = "HOST_BIN_SUFFIX" + KIND = "host" + TARGET_SUBST_VAR = "RUST_HOST_TARGET" + + +class RustTests(ContextDerived): + __slots__ = ("names", "features", "output_category") + + def __init__(self, context, names, features): + ContextDerived.__init__(self, context) + self.names = names + self.features = features + self.output_category = "rusttests" + + +class BaseLibrary(Linkable): + """Generic context derived container object for libraries.""" + + __slots__ = ("basename", "lib_name", "import_name", "refs") + + def __init__(self, context, basename): + Linkable.__init__(self, context) + + self.basename = self.lib_name = basename + if self.lib_name: + self.lib_name = "%s%s%s" % ( + context.config.lib_prefix, + self.lib_name, + context.config.lib_suffix, + ) + self.import_name = self.lib_name + + self.refs = [] + + def __repr__(self): + return "<%s: %s/%s>" % (type(self).__name__, self.relobjdir, self.lib_name) + + @property + def name(self): + return self.lib_name + + +class Library(BaseLibrary): + """Context derived container object for a library""" + + KIND = "target" + __slots__ = () + + def __init__(self, context, basename, real_name=None): + BaseLibrary.__init__(self, context, real_name or basename) + self.basename = basename + + +class StaticLibrary(Library): + """Context derived container object for a static library""" + + __slots__ = ("link_into", "no_expand_lib") + + def __init__( + self, context, basename, real_name=None, link_into=None, no_expand_lib=False + ): + Library.__init__(self, context, basename, real_name) + self.link_into = link_into + self.no_expand_lib = no_expand_lib + + +class SandboxedWasmLibrary(Library): + """Context derived container object for a static sandboxed wasm library""" + + # This is a real static library; make it known to the build system. + no_expand_lib = True + KIND = "wasm" + + def __init__(self, context, basename, real_name=None): + Library.__init__(self, context, basename, real_name) + + # Wasm libraries are not going to compile unless we have a compiler + # for them. + assert context.config.substs["WASM_CC"] and context.config.substs["WASM_CXX"] + + self.lib_name = "%s%s%s" % ( + context.config.dll_prefix, + real_name or basename, + context.config.dll_suffix, + ) + + def _obj_suffix(self): + """Can be overridden by a base class for custom behavior.""" + return self.config.substs.get("WASM_OBJ_SUFFIX", "") + + +class BaseRustLibrary(object): + slots = ( + "cargo_file", + "crate_type", + "dependencies", + "deps_path", + "features", + "output_category", + "is_gkrust", + ) + + def init( + self, + context, + basename, + cargo_file, + crate_type, + dependencies, + features, + is_gkrust, + ): + self.is_gkrust = is_gkrust + self.cargo_file = cargo_file + self.crate_type = crate_type + # We need to adjust our naming here because cargo replaces '-' in + # package names defined in Cargo.toml with underscores in actual + # filenames. But we need to keep the basename consistent because + # many other things in the build system depend on that. + assert self.crate_type == "staticlib" + self.lib_name = "%s%s%s" % ( + context.config.lib_prefix, + basename.replace("-", "_"), + context.config.lib_suffix, + ) + self.dependencies = dependencies + self.features = features + self.output_category = context.get("RUST_LIBRARY_OUTPUT_CATEGORY") + # Skip setting properties below which depend on cargo + # when we don't have a compile environment. The required + # config keys won't be available, but the instance variables + # that we don't set should never be accessed by the actual + # build in that case. + if not context.config.substs.get("COMPILE_ENVIRONMENT"): + return + build_dir = mozpath.join( + context.config.topobjdir, + cargo_output_directory(context, self.TARGET_SUBST_VAR), + ) + self.import_name = mozpath.join(build_dir, self.lib_name) + self.deps_path = mozpath.join(build_dir, "deps") + + +class RustLibrary(StaticLibrary, BaseRustLibrary): + """Context derived container object for a rust static library""" + + KIND = "target" + TARGET_SUBST_VAR = "RUST_TARGET" + FEATURES_VAR = "RUST_LIBRARY_FEATURES" + LIB_FILE_VAR = "RUST_LIBRARY_FILE" + __slots__ = BaseRustLibrary.slots + + def __init__( + self, + context, + basename, + cargo_file, + crate_type, + dependencies, + features, + is_gkrust=False, + link_into=None, + ): + StaticLibrary.__init__( + self, + context, + basename, + link_into=link_into, + # A rust library is a real static library ; make + # it known to the build system. + no_expand_lib=True, + ) + BaseRustLibrary.init( + self, + context, + basename, + cargo_file, + crate_type, + dependencies, + features, + is_gkrust, + ) + + +class SharedLibrary(Library): + """Context derived container object for a shared library""" + + __slots__ = ( + "soname", + "variant", + "symbols_file", + "output_category", + "symbols_link_arg", + ) + + DICT_ATTRS = { + "basename", + "import_name", + "install_target", + "lib_name", + "relobjdir", + "soname", + } + + FRAMEWORK = 1 + MAX_VARIANT = 2 + + def __init__( + self, + context, + basename, + real_name=None, + soname=None, + variant=None, + symbols_file=False, + ): + assert variant in range(1, self.MAX_VARIANT) or variant is None + Library.__init__(self, context, basename, real_name) + self.variant = variant + self.lib_name = real_name or basename + self.output_category = context.get("SHARED_LIBRARY_OUTPUT_CATEGORY") + assert self.lib_name + + if variant == self.FRAMEWORK: + self.import_name = self.lib_name + else: + self.import_name = "%s%s%s" % ( + context.config.import_prefix, + self.lib_name, + context.config.import_suffix, + ) + self.lib_name = "%s%s%s" % ( + context.config.dll_prefix, + self.lib_name, + context.config.dll_suffix, + ) + if soname: + self.soname = "%s%s%s" % ( + context.config.dll_prefix, + soname, + context.config.dll_suffix, + ) + else: + self.soname = self.lib_name + + if symbols_file is False: + # No symbols file. + self.symbols_file = None + elif symbols_file is True: + # Symbols file with default name. + if context.config.substs["OS_TARGET"] == "WINNT": + self.symbols_file = "%s.def" % self.lib_name + else: + self.symbols_file = "%s.symbols" % self.lib_name + else: + # Explicitly provided name. + self.symbols_file = symbols_file + + if self.symbols_file: + os_target = context.config.substs["OS_TARGET"] + if os_target == "Darwin": + self.symbols_link_arg = ( + "-Wl,-exported_symbols_list," + self.symbols_file + ) + elif os_target == "SunOS": + self.symbols_link_arg = ( + "-z gnu-version-script-compat -Wl,--version-script," + + self.symbols_file + ) + elif os_target == "WINNT": + if context.config.substs.get("GNU_CC"): + self.symbols_link_arg = self.symbols_file + else: + self.symbols_link_arg = "-DEF:" + self.symbols_file + elif context.config.substs.get("GCC_USE_GNU_LD"): + self.symbols_link_arg = "-Wl,--version-script," + self.symbols_file + + +class HostSharedLibrary(HostMixin, Library): + """Context derived container object for a host shared library. + + This class supports less things than SharedLibrary does for target shared + libraries. Currently has enough build system support to build the clang + plugin.""" + + KIND = "host" + + def __init__(self, context, basename): + Library.__init__(self, context, basename) + self.lib_name = "%s%s%s" % ( + context.config.host_dll_prefix, + self.basename, + context.config.host_dll_suffix, + ) + + +class ExternalLibrary(object): + """Empty mixin for libraries built by an external build system.""" + + +class ExternalStaticLibrary(StaticLibrary, ExternalLibrary): + """Context derived container for static libraries built by an external + build system.""" + + +class ExternalSharedLibrary(SharedLibrary, ExternalLibrary): + """Context derived container for shared libraries built by an external + build system.""" + + +class HostLibrary(HostMixin, BaseLibrary): + """Context derived container object for a host library""" + + KIND = "host" + no_expand_lib = False + + +class HostRustLibrary(HostLibrary, BaseRustLibrary): + """Context derived container object for a host rust library""" + + KIND = "host" + TARGET_SUBST_VAR = "RUST_HOST_TARGET" + FEATURES_VAR = "HOST_RUST_LIBRARY_FEATURES" + LIB_FILE_VAR = "HOST_RUST_LIBRARY_FILE" + __slots__ = BaseRustLibrary.slots + no_expand_lib = True + + def __init__( + self, + context, + basename, + cargo_file, + crate_type, + dependencies, + features, + is_gkrust, + ): + HostLibrary.__init__(self, context, basename) + BaseRustLibrary.init( + self, + context, + basename, + cargo_file, + crate_type, + dependencies, + features, + is_gkrust, + ) + + +class TestManifest(ContextDerived): + """Represents a manifest file containing information about tests.""" + + __slots__ = ( + # The type of test manifest this is. + "flavor", + # Maps source filename to destination filename. The destination + # path is relative from the tests root directory. Values are 2-tuples + # of (destpath, is_test_file) where the 2nd item is True if this + # item represents a test file (versus a support file). + "installs", + # A list of pattern matching installs to perform. Entries are + # (base, pattern, dest). + "pattern_installs", + # Where all files for this manifest flavor are installed in the unified + # test package directory. + "install_prefix", + # Set of files provided by an external mechanism. + "external_installs", + # Set of files required by multiple test directories, whose installation + # will be resolved when running tests. + "deferred_installs", + # The full path of this manifest file. + "path", + # The directory where this manifest is defined. + "directory", + # The parsed manifestparser.TestManifest instance. + "manifest", + # List of tests. Each element is a dict of metadata. + "tests", + # The relative path of the parsed manifest within the srcdir. + "manifest_relpath", + # The relative path of the parsed manifest within the objdir. + "manifest_obj_relpath", + # The relative paths to all source files for this manifest. + "source_relpaths", + # If this manifest is a duplicate of another one, this is the + # manifestparser.TestManifest of the other one. + "dupe_manifest", + ) + + def __init__( + self, + context, + path, + manifest, + flavor=None, + install_prefix=None, + relpath=None, + sources=(), + dupe_manifest=False, + ): + ContextDerived.__init__(self, context) + + assert flavor in all_test_flavors() + + self.path = path + self.directory = mozpath.dirname(path) + self.manifest = manifest + self.flavor = flavor + self.install_prefix = install_prefix + self.manifest_relpath = relpath + self.manifest_obj_relpath = relpath + self.source_relpaths = sources + self.dupe_manifest = dupe_manifest + self.installs = {} + self.pattern_installs = [] + self.tests = [] + self.external_installs = set() + self.deferred_installs = set() + + +class LocalInclude(ContextDerived): + """Describes an individual local include path.""" + + __slots__ = ("path",) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.path = path + + +class PerSourceFlag(ContextDerived): + """Describes compiler flags specified for individual source files.""" + + __slots__ = ("file_name", "flags") + + def __init__(self, context, file_name, flags): + ContextDerived.__init__(self, context) + + self.file_name = file_name + self.flags = flags + + +class JARManifest(ContextDerived): + """Describes an individual JAR manifest file and how to process it. + + This class isn't very useful for optimizing backends yet because we don't + capture defines. We can't capture defines safely until all of them are + defined in moz.build and not Makefile.in files. + """ + + __slots__ = ("path",) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.path = path + + +class BaseSources(ContextDerived): + """Base class for files to be compiled during the build.""" + + __slots__ = ("files", "static_files", "generated_files", "canonical_suffix") + + def __init__(self, context, static_files, generated_files, canonical_suffix): + ContextDerived.__init__(self, context) + + # Sorted so output is consistent and we don't bump mtimes, but always + # order generated files after static ones to be consistent across build + # environments, which may have different objdir paths relative to + # topsrcdir. + self.static_files = sorted(static_files) + self.generated_files = sorted(generated_files) + self.files = self.static_files + self.generated_files + self.canonical_suffix = canonical_suffix + + +class Sources(BaseSources): + """Represents files to be compiled during the build.""" + + def __init__(self, context, static_files, generated_files, canonical_suffix): + BaseSources.__init__( + self, context, static_files, generated_files, canonical_suffix + ) + + +class PgoGenerateOnlySources(BaseSources): + """Represents files to be compiled during the build. + + These files are only used during the PGO generation phase.""" + + def __init__(self, context, files): + BaseSources.__init__(self, context, files, [], ".cpp") + + +class HostSources(HostMixin, BaseSources): + """Represents files to be compiled for the host during the build.""" + + def __init__(self, context, static_files, generated_files, canonical_suffix): + BaseSources.__init__( + self, context, static_files, generated_files, canonical_suffix + ) + + +class WasmSources(BaseSources): + """Represents files to be compiled with the wasm compiler during the build.""" + + def __init__(self, context, static_files, generated_files, canonical_suffix): + BaseSources.__init__( + self, context, static_files, generated_files, canonical_suffix + ) + + +class UnifiedSources(BaseSources): + """Represents files to be compiled in a unified fashion during the build.""" + + __slots__ = ("have_unified_mapping", "unified_source_mapping") + + def __init__(self, context, static_files, generated_files, canonical_suffix): + BaseSources.__init__( + self, context, static_files, generated_files, canonical_suffix + ) + + unified_build = context.config.substs.get("ENABLE_UNIFIED_BUILD", False) + files_per_unified_file = ( + context.get("FILES_PER_UNIFIED_FILE", 16) if unified_build else 1 + ) + + self.have_unified_mapping = files_per_unified_file > 1 + + if self.have_unified_mapping: + # On Windows, path names have a maximum length of 255 characters, + # so avoid creating extremely long path names. + unified_prefix = context.relsrcdir + if len(unified_prefix) > 20: + unified_prefix = unified_prefix[-20:].split("/", 1)[-1] + unified_prefix = unified_prefix.replace("/", "_") + + suffix = self.canonical_suffix[1:] + unified_prefix = "Unified_%s_%s" % (suffix, unified_prefix) + self.unified_source_mapping = list( + group_unified_files( + # NOTE: self.files is already (partially) sorted, and we + # intentionally do not re-sort it here to avoid a dependency + # on the build environment's objdir path. + self.files, + unified_prefix=unified_prefix, + unified_suffix=suffix, + files_per_unified_file=files_per_unified_file, + ) + ) + + +class InstallationTarget(ContextDerived): + """Describes the rules that affect where files get installed to.""" + + __slots__ = ("xpiname", "subdir", "target", "enabled") + + def __init__(self, context): + ContextDerived.__init__(self, context) + + self.xpiname = context.get("XPI_NAME", "") + self.subdir = context.get("DIST_SUBDIR", "") + self.target = context["FINAL_TARGET"] + self.enabled = context["DIST_INSTALL"] is not False + + def is_custom(self): + """Returns whether or not the target is not derived from the default + given xpiname and subdir.""" + + return ( + FinalTargetValue(dict(XPI_NAME=self.xpiname, DIST_SUBDIR=self.subdir)) + == self.target + ) + + +class FinalTargetFiles(ContextDerived): + """Sandbox container object for FINAL_TARGET_FILES, which is a + HierarchicalStringList. + + We need an object derived from ContextDerived for use in the backend, so + this object fills that role. It just has a reference to the underlying + HierarchicalStringList, which is created when parsing FINAL_TARGET_FILES. + """ + + __slots__ = "files" + + def __init__(self, sandbox, files): + ContextDerived.__init__(self, sandbox) + self.files = files + + +class FinalTargetPreprocessedFiles(ContextDerived): + """Sandbox container object for FINAL_TARGET_PP_FILES, which is a + HierarchicalStringList. + + We need an object derived from ContextDerived for use in the backend, so + this object fills that role. It just has a reference to the underlying + HierarchicalStringList, which is created when parsing + FINAL_TARGET_PP_FILES. + """ + + __slots__ = "files" + + def __init__(self, sandbox, files): + ContextDerived.__init__(self, sandbox) + self.files = files + + +class LocalizedFiles(FinalTargetFiles): + """Sandbox container object for LOCALIZED_FILES, which is a + HierarchicalStringList. + """ + + pass + + +class LocalizedPreprocessedFiles(FinalTargetPreprocessedFiles): + """Sandbox container object for LOCALIZED_PP_FILES, which is a + HierarchicalStringList. + """ + + pass + + +class ObjdirFiles(FinalTargetFiles): + """Sandbox container object for OBJDIR_FILES, which is a + HierarchicalStringList. + """ + + @property + def install_target(self): + return "" + + +class ObjdirPreprocessedFiles(FinalTargetPreprocessedFiles): + """Sandbox container object for OBJDIR_PP_FILES, which is a + HierarchicalStringList. + """ + + @property + def install_target(self): + return "" + + +class TestHarnessFiles(FinalTargetFiles): + """Sandbox container object for TEST_HARNESS_FILES, + which is a HierarchicalStringList. + """ + + @property + def install_target(self): + return "_tests" + + +class Exports(FinalTargetFiles): + """Context derived container object for EXPORTS, which is a + HierarchicalStringList. + + We need an object derived from ContextDerived for use in the backend, so + this object fills that role. It just has a reference to the underlying + HierarchicalStringList, which is created when parsing EXPORTS. + """ + + @property + def install_target(self): + return "dist/include" + + +class GeneratedFile(ContextDerived): + """Represents a generated file.""" + + __slots__ = ( + "script", + "method", + "outputs", + "inputs", + "flags", + "required_before_export", + "required_before_compile", + "required_during_compile", + "localized", + "force", + "py2", + ) + + def __init__( + self, + context, + script, + method, + outputs, + inputs, + flags=(), + localized=False, + force=False, + py2=False, + required_during_compile=None, + ): + ContextDerived.__init__(self, context) + self.script = script + self.method = method + self.outputs = outputs if isinstance(outputs, tuple) else (outputs,) + self.inputs = inputs + self.flags = flags + self.localized = localized + self.force = force + self.py2 = py2 + + if self.config.substs.get("MOZ_WIDGET_TOOLKIT") == "android": + # In GeckoView builds we process Jinja files during pre-export + self.required_before_export = [ + f for f in self.inputs if f.endswith(".jinja") + ] + else: + self.required_before_export = False + + suffixes = [ + ".h", + ".py", + ".rs", + # We need to compile Java to generate JNI wrappers for native code + # compilation to consume. + "android_apks", + ".profdata", + ".webidl", + ] + + try: + lib_suffix = context.config.substs["LIB_SUFFIX"] + suffixes.append("." + lib_suffix) + except KeyError: + # Tests may not define LIB_SUFFIX + pass + + suffixes = tuple(suffixes) + + self.required_before_compile = [ + f + for f in self.outputs + if f.endswith(suffixes) or "stl_wrappers/" in f or "xpidl.stub" in f + ] + + if required_during_compile is None: + self.required_during_compile = [ + f + for f in self.outputs + if f.endswith( + (".asm", ".c", ".cpp", ".inc", ".m", ".mm", ".def", "symverscript") + ) + ] + else: + self.required_during_compile = required_during_compile + + +class ChromeManifestEntry(ContextDerived): + """Represents a chrome.manifest entry.""" + + __slots__ = ("path", "entry") + + def __init__(self, context, manifest_path, entry): + ContextDerived.__init__(self, context) + assert isinstance(entry, ManifestEntry) + self.path = mozpath.join(self.install_target, manifest_path) + # Ensure the entry is relative to the directory containing the + # manifest path. + entry = entry.rebase(mozpath.dirname(manifest_path)) + # Then add the install_target to the entry base directory. + self.entry = entry.move(mozpath.dirname(self.path)) |