diff options
Diffstat (limited to 'python/mozbuild/mozbuild/frontend/gyp_reader.py')
| -rw-r--r-- | python/mozbuild/mozbuild/frontend/gyp_reader.py | 497 |
1 files changed, 497 insertions, 0 deletions
diff --git a/python/mozbuild/mozbuild/frontend/gyp_reader.py b/python/mozbuild/mozbuild/frontend/gyp_reader.py new file mode 100644 index 0000000000..cd69dfddce --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/gyp_reader.py @@ -0,0 +1,497 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys +import time + +import gyp +import gyp.msvs_emulation +import mozpack.path as mozpath +import six +from mozpack.files import FileFinder + +from mozbuild import shellutil +from mozbuild.util import expand_variables + +from .context import VARIABLES, ObjDirPath, SourcePath, TemplateContext +from .sandbox import alphabetical_sorted + +# Define this module as gyp.generator.mozbuild so that gyp can use it +# as a generator under the name "mozbuild". +sys.modules["gyp.generator.mozbuild"] = sys.modules[__name__] + +# build/gyp_chromium does this: +# script_dir = os.path.dirname(os.path.realpath(__file__)) +# chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir)) +# sys.path.insert(0, os.path.join(chrome_src, 'tools', 'gyp', 'pylib')) +# We're not importing gyp_chromium, but we want both script_dir and +# chrome_src for the default includes, so go backwards from the pylib +# directory, which is the parent directory of gyp module. +chrome_src = mozpath.abspath( + mozpath.join(mozpath.dirname(gyp.__file__), "../../../../..") +) +script_dir = mozpath.join(chrome_src, "build") + + +# Default variables gyp uses when evaluating gyp files. +generator_default_variables = {} +for dirname in [ + "INTERMEDIATE_DIR", + "SHARED_INTERMEDIATE_DIR", + "PRODUCT_DIR", + "LIB_DIR", + "SHARED_LIB_DIR", +]: + # Some gyp steps fail if these are empty(!). + generator_default_variables[dirname] = "$" + dirname + +for unused in [ + "RULE_INPUT_PATH", + "RULE_INPUT_ROOT", + "RULE_INPUT_NAME", + "RULE_INPUT_DIRNAME", + "RULE_INPUT_EXT", + "EXECUTABLE_PREFIX", + "EXECUTABLE_SUFFIX", + "STATIC_LIB_PREFIX", + "STATIC_LIB_SUFFIX", + "SHARED_LIB_PREFIX", + "SHARED_LIB_SUFFIX", + "LINKER_SUPPORTS_ICF", +]: + generator_default_variables[unused] = "" + + +class GypContext(TemplateContext): + """Specialized Context for use with data extracted from Gyp. + + config is the ConfigEnvironment for this context. + relobjdir is the object directory that will be used for this context, + relative to the topobjdir defined in the ConfigEnvironment. + """ + + def __init__(self, config, relobjdir): + self._relobjdir = relobjdir + TemplateContext.__init__( + self, template="Gyp", allowed_variables=VARIABLES, config=config + ) + + +def handle_actions(actions, context, action_overrides): + idir = "$INTERMEDIATE_DIR/" + for action in actions: + name = action["action_name"] + if name not in action_overrides: + raise RuntimeError("GYP action %s not listed in action_overrides" % name) + outputs = action["outputs"] + if len(outputs) > 1: + raise NotImplementedError( + "GYP actions with more than one output not supported: %s" % name + ) + output = outputs[0] + if not output.startswith(idir): + raise NotImplementedError( + "GYP actions outputting to somewhere other than " + "<(INTERMEDIATE_DIR) not supported: %s" % output + ) + output = output[len(idir) :] + context["GENERATED_FILES"] += [output] + g = context["GENERATED_FILES"][output] + g.script = action_overrides[name] + g.inputs = action["inputs"] + + +def handle_copies(copies, context): + dist = "$PRODUCT_DIR/dist/" + for copy in copies: + dest = copy["destination"] + if not dest.startswith(dist): + raise NotImplementedError( + "GYP copies to somewhere other than <(PRODUCT_DIR)/dist not supported: %s" + % dest + ) + dest_paths = dest[len(dist) :].split("/") + exports = context["EXPORTS"] + while dest_paths: + exports = getattr(exports, dest_paths.pop(0)) + exports += sorted(copy["files"], key=lambda x: x.lower()) + + +def process_gyp_result( + gyp_result, + gyp_dir_attrs, + path, + config, + output, + non_unified_sources, + action_overrides, +): + flat_list, targets, data = gyp_result + no_chromium = gyp_dir_attrs.no_chromium + no_unified = gyp_dir_attrs.no_unified + + # Process all targets from the given gyp files and its dependencies. + # The path given to AllTargets needs to use os.sep, while the frontend code + # gives us paths normalized with forward slash separator. + for target in sorted( + gyp.common.AllTargets(flat_list, targets, path.replace("/", os.sep)) + ): + build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(target) + + # Each target is given its own objdir. The base of that objdir + # is derived from the relative path from the root gyp file path + # to the current build_file, placed under the given output + # directory. Since several targets can be in a given build_file, + # separate them in subdirectories using the build_file basename + # and the target_name. + reldir = mozpath.relpath(mozpath.dirname(build_file), mozpath.dirname(path)) + subdir = "%s_%s" % ( + mozpath.splitext(mozpath.basename(build_file))[0], + target_name, + ) + # Emit a context for each target. + context = GypContext( + config, + mozpath.relpath(mozpath.join(output, reldir, subdir), config.topobjdir), + ) + context.add_source(mozpath.abspath(build_file)) + # The list of included files returned by gyp are relative to build_file + for f in data[build_file]["included_files"]: + context.add_source( + mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)) + ) + + spec = targets[target] + + # Derive which gyp configuration to use based on MOZ_DEBUG. + c = "Debug" if config.substs.get("MOZ_DEBUG") else "Release" + if c not in spec["configurations"]: + raise RuntimeError( + "Missing %s gyp configuration for target %s " + "in %s" % (c, target_name, build_file) + ) + target_conf = spec["configurations"][c] + + if "actions" in spec: + handle_actions(spec["actions"], context, action_overrides) + if "copies" in spec: + handle_copies(spec["copies"], context) + + use_libs = [] + libs = [] + + def add_deps(s): + for t in s.get("dependencies", []) + s.get("dependencies_original", []): + ty = targets[t]["type"] + if ty in ("static_library", "shared_library"): + l = targets[t]["target_name"] + if l not in use_libs: + use_libs.append(l) + # Manually expand out transitive dependencies-- + # gyp won't do this for static libs or none targets. + if ty in ("static_library", "none"): + add_deps(targets[t]) + libs.extend(spec.get("libraries", [])) + + # XXX: this sucks, but webrtc breaks with this right now because + # it builds a library called 'gtest' and we just get lucky + # that it isn't in USE_LIBS by that name anywhere. + if no_chromium: + add_deps(spec) + + os_libs = [] + for l in libs: + if l.startswith("-"): + if l.startswith("-l"): + # Remove "-l" for consumption in OS_LIBS. Other flags + # are passed through unchanged. + l = l[2:] + if l not in os_libs: + os_libs.append(l) + elif l.endswith(".lib"): + l = l[:-4] + if l not in os_libs: + os_libs.append(l) + elif l: + # For library names passed in from moz.build. + l = os.path.basename(l) + if l not in use_libs: + use_libs.append(l) + + if spec["type"] == "none": + if not ("actions" in spec or "copies" in spec): + continue + elif spec["type"] in ("static_library", "shared_library", "executable"): + # Remove leading 'lib' from the target_name if any, and use as + # library name. + name = six.ensure_text(spec["target_name"]) + if spec["type"] in ("static_library", "shared_library"): + if name.startswith("lib"): + name = name[3:] + context["LIBRARY_NAME"] = name + else: + context["PROGRAM"] = name + if spec["type"] == "shared_library": + context["FORCE_SHARED_LIB"] = True + elif ( + spec["type"] == "static_library" + and spec.get("variables", {}).get("no_expand_libs", "0") == "1" + ): + # PSM links a NSS static library, but our folded libnss + # doesn't actually export everything that all of the + # objects within would need, so that one library + # should be built as a real static library. + context["NO_EXPAND_LIBS"] = True + if use_libs: + context["USE_LIBS"] = sorted(use_libs, key=lambda s: s.lower()) + if os_libs: + context["OS_LIBS"] = os_libs + # gyp files contain headers and asm sources in sources lists. + sources = [] + unified_sources = [] + extensions = set() + use_defines_in_asflags = False + for f in spec.get("sources", []): + ext = mozpath.splitext(f)[-1] + extensions.add(ext) + if f.startswith("$INTERMEDIATE_DIR/"): + s = ObjDirPath(context, f.replace("$INTERMEDIATE_DIR/", "!")) + else: + s = SourcePath(context, f) + if ext == ".h": + continue + if ext == ".def": + context["SYMBOLS_FILE"] = s + elif ext != ".S" and not no_unified and s not in non_unified_sources: + unified_sources.append(s) + else: + sources.append(s) + # The Mozilla build system doesn't use DEFINES for building + # ASFILES. + if ext == ".s": + use_defines_in_asflags = True + + # The context expects alphabetical order when adding sources + context["SOURCES"] = alphabetical_sorted(sources) + context["UNIFIED_SOURCES"] = alphabetical_sorted(unified_sources) + + defines = target_conf.get("defines", []) + if config.substs["CC_TYPE"] == "clang-cl" and no_chromium: + msvs_settings = gyp.msvs_emulation.MsvsSettings(spec, {}) + # Hack: MsvsSettings._TargetConfig tries to compare a str to an int, + # so convert manually. + msvs_settings.vs_version.short_name = int( + msvs_settings.vs_version.short_name + ) + defines.extend(msvs_settings.GetComputedDefines(c)) + for define in defines: + if "=" in define: + name, value = define.split("=", 1) + context["DEFINES"][name] = value + else: + context["DEFINES"][define] = True + + product_dir_dist = "$PRODUCT_DIR/dist/" + for include in target_conf.get("include_dirs", []): + if include.startswith(product_dir_dist): + # special-case includes of <(PRODUCT_DIR)/dist/ to match + # handle_copies above. This is used for NSS' exports. + include = "!/dist/include/" + include[len(product_dir_dist) :] + elif include.startswith(config.topobjdir): + # NSPR_INCLUDE_DIR gets passed into the NSS build this way. + include = "!/" + mozpath.relpath(include, config.topobjdir) + else: + # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do. + # + # NB: gyp files sometimes have actual absolute paths (e.g. + # /usr/include32) and sometimes paths that moz.build considers + # absolute, i.e. starting from topsrcdir. There's no good way + # to tell them apart here, and the actual absolute paths are + # likely bogus. In any event, actual absolute paths will be + # filtered out by trying to find them in topsrcdir. + # + # We do allow !- and %-prefixed paths, assuming they come + # from moz.build and will be handled the same way as if they + # were given to LOCAL_INCLUDES in moz.build. + if include.startswith("/"): + resolved = mozpath.abspath( + mozpath.join(config.topsrcdir, include[1:]) + ) + elif not include.startswith(("!", "%")): + resolved = mozpath.abspath( + mozpath.join(mozpath.dirname(build_file), include) + ) + if not include.startswith(("!", "%")) and not os.path.exists( + resolved + ): + continue + context["LOCAL_INCLUDES"] += [include] + + context["ASFLAGS"] = target_conf.get("asflags_mozilla", []) + if use_defines_in_asflags and defines: + context["ASFLAGS"] += ["-D" + d for d in defines] + if config.substs["OS_TARGET"] == "SunOS": + context["LDFLAGS"] = target_conf.get("ldflags", []) + flags = target_conf.get("cflags_mozilla", []) + if flags: + suffix_map = { + ".c": "CFLAGS", + ".cpp": "CXXFLAGS", + ".cc": "CXXFLAGS", + ".m": "CMFLAGS", + ".mm": "CMMFLAGS", + } + variables = (suffix_map[e] for e in extensions if e in suffix_map) + for var in variables: + for f in flags: + # We may be getting make variable references out of the + # gyp data, and we don't want those in emitted data, so + # substitute them with their actual value. + f = expand_variables(f, config.substs).split() + if not f: + continue + # the result may be a string or a list. + if isinstance(f, six.string_types): + context[var].append(f) + else: + context[var].extend(f) + else: + # Ignore other types because we don't have + # anything using them, and we're not testing them. They can be + # added when that becomes necessary. + raise NotImplementedError("Unsupported gyp target type: %s" % spec["type"]) + + if not no_chromium: + # Add some features to all contexts. Put here in case LOCAL_INCLUDES + # order matters. + context["LOCAL_INCLUDES"] += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + ] + # These get set via VC project file settings for normal GYP builds. + if config.substs["OS_TARGET"] == "WINNT": + context["DEFINES"]["UNICODE"] = True + context["DEFINES"]["_UNICODE"] = True + context["COMPILE_FLAGS"]["OS_INCLUDES"] = [] + + for key, value in gyp_dir_attrs.sandbox_vars.items(): + if context.get(key) and isinstance(context[key], list): + # If we have a key from sanbox_vars that's also been + # populated here we use the value from sandbox_vars as our + # basis rather than overriding outright. + context[key] = value + context[key] + elif context.get(key) and isinstance(context[key], dict): + context[key].update(value) + else: + context[key] = value + + yield context + + +# A version of gyp.Load that doesn't return the generator (because module objects +# aren't Pickle-able, and we don't use it anyway). +def load_gyp(*args): + _, flat_list, targets, data = gyp.Load(*args) + return flat_list, targets, data + + +class GypProcessor(object): + """Reads a gyp configuration in the background using the given executor and + emits GypContexts for the backend to process. + + config is a ConfigEnvironment, path is the path to a root gyp configuration + file, and output is the base path under which the objdir for the various + gyp dependencies will be. gyp_dir_attrs are attributes set for the dir + from moz.build. + """ + + def __init__( + self, + config, + gyp_dir_attrs, + path, + output, + executor, + action_overrides, + non_unified_sources, + ): + self._path = path + self._config = config + self._output = output + self._non_unified_sources = non_unified_sources + self._gyp_dir_attrs = gyp_dir_attrs + self._action_overrides = action_overrides + self.execution_time = 0.0 + self._results = [] + + # gyp expects plain str instead of unicode. The frontend code gives us + # unicode strings, so convert them. + if config.substs["CC_TYPE"] == "clang-cl": + # This isn't actually used anywhere in this generator, but it's needed + # to override the registry detection of VC++ in gyp. + os.environ.update( + { + "GYP_MSVS_OVERRIDE_PATH": "fake_path", + "GYP_MSVS_VERSION": config.substs["MSVS_VERSION"], + } + ) + + params = { + "parallel": False, + "generator_flags": {}, + "build_files": [path], + "root_targets": None, + } + # The NSS gyp configuration uses CC and CFLAGS to determine the + # floating-point ABI on arm. + os.environ.update( + CC=config.substs["CC"], + CFLAGS=shellutil.quote(*config.substs["CC_BASE_FLAGS"]), + ) + + if gyp_dir_attrs.no_chromium: + includes = [] + depth = mozpath.dirname(path) + else: + depth = chrome_src + # Files that gyp_chromium always includes + includes = [mozpath.join(script_dir, "gyp_includes", "common.gypi")] + finder = FileFinder(chrome_src) + includes.extend( + mozpath.join(chrome_src, name) + for name, _ in finder.find("*/supplement.gypi") + ) + + str_vars = dict(gyp_dir_attrs.variables) + str_vars["python"] = sys.executable + self._gyp_loader_future = executor.submit( + load_gyp, [path], "mozbuild", str_vars, includes, depth, params + ) + + @property + def results(self): + if self._results: + for res in self._results: + yield res + else: + # We report our execution time as the time spent blocked in a call + # to `result`, which is the only case a gyp processor will + # contribute significantly to total wall time. + t0 = time.monotonic() + flat_list, targets, data = self._gyp_loader_future.result() + self.execution_time += time.monotonic() - t0 + results = [] + for res in process_gyp_result( + (flat_list, targets, data), + self._gyp_dir_attrs, + self._path, + self._config, + self._output, + self._non_unified_sources, + self._action_overrides, + ): + results.append(res) + yield res + self._results = results |