diff options
Diffstat (limited to '')
-rw-r--r-- | compiler/rustc_llvm/Cargo.toml | 14 | ||||
-rw-r--r-- | compiler/rustc_llvm/build.rs | 368 | ||||
-rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/.editorconfig | 6 | ||||
-rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp | 222 | ||||
-rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp | 117 | ||||
-rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h | 121 | ||||
-rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/Linker.cpp | 48 | ||||
-rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp | 1763 | ||||
-rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/README | 16 | ||||
-rw-r--r-- | compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 1972 | ||||
-rw-r--r-- | compiler/rustc_llvm/src/lib.rs | 188 |
11 files changed, 4835 insertions, 0 deletions
diff --git a/compiler/rustc_llvm/Cargo.toml b/compiler/rustc_llvm/Cargo.toml new file mode 100644 index 000000000..34556df3c --- /dev/null +++ b/compiler/rustc_llvm/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rustc_llvm" +version = "0.0.0" +edition = "2021" + +[features] +static-libstdcpp = [] +emscripten = [] + +[dependencies] +libc = "0.2.73" + +[build-dependencies] +cc = "1.0.69" diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs new file mode 100644 index 000000000..62ef5804d --- /dev/null +++ b/compiler/rustc_llvm/build.rs @@ -0,0 +1,368 @@ +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fmt::Display; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +const OPTIONAL_COMPONENTS: &[&str] = &[ + "x86", + "arm", + "aarch64", + "amdgpu", + "avr", + "m68k", + "mips", + "powerpc", + "systemz", + "jsbackend", + "webassembly", + "msp430", + "sparc", + "nvptx", + "hexagon", + "riscv", + "bpf", +]; + +const REQUIRED_COMPONENTS: &[&str] = + &["ipo", "bitreader", "bitwriter", "linker", "asmparser", "lto", "coverage", "instrumentation"]; + +fn detect_llvm_link() -> (&'static str, &'static str) { + // Force the link mode we want, preferring static by default, but + // possibly overridden by `configure --enable-llvm-link-shared`. + if tracked_env_var_os("LLVM_LINK_SHARED").is_some() { + ("dylib", "--link-shared") + } else { + ("static", "--link-static") + } +} + +// Because Cargo adds the compiler's dylib path to our library search path, llvm-config may +// break: the dylib path for the compiler, as of this writing, contains a copy of the LLVM +// shared library, which means that when our freshly built llvm-config goes to load it's +// associated LLVM, it actually loads the compiler's LLVM. In particular when building the first +// compiler (i.e., in stage 0) that's a problem, as the compiler's LLVM is likely different from +// the one we want to use. As such, we restore the environment to what bootstrap saw. This isn't +// perfect -- we might actually want to see something from Cargo's added library paths -- but +// for now it works. +fn restore_library_path() { + let key = tracked_env_var_os("REAL_LIBRARY_PATH_VAR").expect("REAL_LIBRARY_PATH_VAR"); + if let Some(env) = tracked_env_var_os("REAL_LIBRARY_PATH") { + env::set_var(&key, &env); + } else { + env::remove_var(&key); + } +} + +/// Reads an environment variable and adds it to dependencies. +/// Supposed to be used for all variables except those set for build scripts by cargo +/// <https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts> +fn tracked_env_var_os<K: AsRef<OsStr> + Display>(key: K) -> Option<OsString> { + println!("cargo:rerun-if-env-changed={}", key); + env::var_os(key) +} + +fn rerun_if_changed_anything_in_dir(dir: &Path) { + let mut stack = dir + .read_dir() + .unwrap() + .map(|e| e.unwrap()) + .filter(|e| &*e.file_name() != ".git") + .collect::<Vec<_>>(); + while let Some(entry) = stack.pop() { + let path = entry.path(); + if entry.file_type().unwrap().is_dir() { + stack.extend(path.read_dir().unwrap().map(|e| e.unwrap())); + } else { + println!("cargo:rerun-if-changed={}", path.display()); + } + } +} + +#[track_caller] +fn output(cmd: &mut Command) -> String { + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => { + println!("\n\nfailed to execute command: {:?}\nerror: {}\n\n", cmd, e); + std::process::exit(1); + } + }; + if !output.status.success() { + panic!( + "command did not execute successfully: {:?}\n\ + expected success, got: {}", + cmd, output.status + ); + } + String::from_utf8(output.stdout).unwrap() +} + +fn main() { + for component in REQUIRED_COMPONENTS.iter().chain(OPTIONAL_COMPONENTS.iter()) { + println!("cargo:rustc-check-cfg=values(llvm_component,\"{}\")", component); + } + + if tracked_env_var_os("RUST_CHECK").is_some() { + // If we're just running `check`, there's no need for LLVM to be built. + return; + } + + restore_library_path(); + + let target = env::var("TARGET").expect("TARGET was not set"); + let llvm_config = + tracked_env_var_os("LLVM_CONFIG").map(|x| Some(PathBuf::from(x))).unwrap_or_else(|| { + if let Some(dir) = tracked_env_var_os("CARGO_TARGET_DIR").map(PathBuf::from) { + let to_test = dir + .parent() + .unwrap() + .parent() + .unwrap() + .join(&target) + .join("llvm/bin/llvm-config"); + if Command::new(&to_test).output().is_ok() { + return Some(to_test); + } + } + None + }); + + if let Some(llvm_config) = &llvm_config { + println!("cargo:rerun-if-changed={}", llvm_config.display()); + } + let llvm_config = llvm_config.unwrap_or_else(|| PathBuf::from("llvm-config")); + + // Test whether we're cross-compiling LLVM. This is a pretty rare case + // currently where we're producing an LLVM for a different platform than + // what this build script is currently running on. + // + // In that case, there's no guarantee that we can actually run the target, + // so the build system works around this by giving us the LLVM_CONFIG for + // the host platform. This only really works if the host LLVM and target + // LLVM are compiled the same way, but for us that's typically the case. + // + // We *want* detect this cross compiling situation by asking llvm-config + // what its host-target is. If that's not the TARGET, then we're cross + // compiling. Unfortunately `llvm-config` seems either be buggy, or we're + // misconfiguring it, because the `i686-pc-windows-gnu` build of LLVM will + // report itself with a `--host-target` of `x86_64-pc-windows-gnu`. This + // tricks us into thinking we're doing a cross build when we aren't, so + // havoc ensues. + // + // In any case, if we're cross compiling, this generally just means that we + // can't trust all the output of llvm-config because it might be targeted + // for the host rather than the target. As a result a bunch of blocks below + // are gated on `if !is_crossed` + let target = env::var("TARGET").expect("TARGET was not set"); + let host = env::var("HOST").expect("HOST was not set"); + let is_crossed = target != host; + + let components = output(Command::new(&llvm_config).arg("--components")); + let mut components = components.split_whitespace().collect::<Vec<_>>(); + components.retain(|c| OPTIONAL_COMPONENTS.contains(c) || REQUIRED_COMPONENTS.contains(c)); + + for component in REQUIRED_COMPONENTS { + if !components.contains(component) { + panic!("require llvm component {} but wasn't found", component); + } + } + + for component in components.iter() { + println!("cargo:rustc-cfg=llvm_component=\"{}\"", component); + } + + // Link in our own LLVM shims, compiled with the same flags as LLVM + let mut cmd = Command::new(&llvm_config); + cmd.arg("--cxxflags"); + let cxxflags = output(&mut cmd); + let mut cfg = cc::Build::new(); + cfg.warnings(false); + for flag in cxxflags.split_whitespace() { + // Ignore flags like `-m64` when we're doing a cross build + if is_crossed && flag.starts_with("-m") { + continue; + } + + if flag.starts_with("-flto") { + continue; + } + + // -Wdate-time is not supported by the netbsd cross compiler + if is_crossed && target.contains("netbsd") && flag.contains("date-time") { + continue; + } + + // Include path contains host directory, replace it with target + if is_crossed && flag.starts_with("-I") { + cfg.flag(&flag.replace(&host, &target)); + continue; + } + + cfg.flag(flag); + } + + for component in &components { + let mut flag = String::from("LLVM_COMPONENT_"); + flag.push_str(&component.to_uppercase()); + cfg.define(&flag, None); + } + + if tracked_env_var_os("LLVM_RUSTLLVM").is_some() { + cfg.define("LLVM_RUSTLLVM", None); + } + + if tracked_env_var_os("LLVM_NDEBUG").is_some() { + cfg.define("NDEBUG", None); + cfg.debug(false); + } + + rerun_if_changed_anything_in_dir(Path::new("llvm-wrapper")); + cfg.file("llvm-wrapper/PassWrapper.cpp") + .file("llvm-wrapper/RustWrapper.cpp") + .file("llvm-wrapper/ArchiveWrapper.cpp") + .file("llvm-wrapper/CoverageMappingWrapper.cpp") + .file("llvm-wrapper/Linker.cpp") + .cpp(true) + .cpp_link_stdlib(None) // we handle this below + .compile("llvm-wrapper"); + + let (llvm_kind, llvm_link_arg) = detect_llvm_link(); + + // Link in all LLVM libraries, if we're using the "wrong" llvm-config then + // we don't pick up system libs because unfortunately they're for the host + // of llvm-config, not the target that we're attempting to link. + let mut cmd = Command::new(&llvm_config); + cmd.arg(llvm_link_arg).arg("--libs"); + + if !is_crossed { + cmd.arg("--system-libs"); + } else if target.contains("windows-gnu") { + println!("cargo:rustc-link-lib=shell32"); + println!("cargo:rustc-link-lib=uuid"); + } else if target.contains("netbsd") || target.contains("haiku") || target.contains("darwin") { + println!("cargo:rustc-link-lib=z"); + } + cmd.args(&components); + + for lib in output(&mut cmd).split_whitespace() { + let name = if let Some(stripped) = lib.strip_prefix("-l") { + stripped + } else if let Some(stripped) = lib.strip_prefix('-') { + stripped + } else if Path::new(lib).exists() { + // On MSVC llvm-config will print the full name to libraries, but + // we're only interested in the name part + let name = Path::new(lib).file_name().unwrap().to_str().unwrap(); + name.trim_end_matches(".lib") + } else if lib.ends_with(".lib") { + // Some MSVC libraries just come up with `.lib` tacked on, so chop + // that off + lib.trim_end_matches(".lib") + } else { + continue; + }; + + // Don't need or want this library, but LLVM's CMake build system + // doesn't provide a way to disable it, so filter it here even though we + // may or may not have built it. We don't reference anything from this + // library and it otherwise may just pull in extra dependencies on + // libedit which we don't want + if name == "LLVMLineEditor" { + continue; + } + + let kind = if name.starts_with("LLVM") { llvm_kind } else { "dylib" }; + println!("cargo:rustc-link-lib={}={}", kind, name); + } + + // LLVM ldflags + // + // If we're a cross-compile of LLVM then unfortunately we can't trust these + // ldflags (largely where all the LLVM libs are located). Currently just + // hack around this by replacing the host triple with the target and pray + // that those -L directories are the same! + let mut cmd = Command::new(&llvm_config); + cmd.arg(llvm_link_arg).arg("--ldflags"); + for lib in output(&mut cmd).split_whitespace() { + if is_crossed { + if let Some(stripped) = lib.strip_prefix("-LIBPATH:") { + println!("cargo:rustc-link-search=native={}", stripped.replace(&host, &target)); + } else if let Some(stripped) = lib.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={}", stripped.replace(&host, &target)); + } + } else if let Some(stripped) = lib.strip_prefix("-LIBPATH:") { + println!("cargo:rustc-link-search=native={}", stripped); + } else if let Some(stripped) = lib.strip_prefix("-l") { + println!("cargo:rustc-link-lib={}", stripped); + } else if let Some(stripped) = lib.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={}", stripped); + } + } + + // Some LLVM linker flags (-L and -l) may be needed even when linking + // rustc_llvm, for example when using static libc++, we may need to + // manually specify the library search path and -ldl -lpthread as link + // dependencies. + let llvm_linker_flags = tracked_env_var_os("LLVM_LINKER_FLAGS"); + if let Some(s) = llvm_linker_flags { + for lib in s.into_string().unwrap().split_whitespace() { + if let Some(stripped) = lib.strip_prefix("-l") { + println!("cargo:rustc-link-lib={}", stripped); + } else if let Some(stripped) = lib.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={}", stripped); + } + } + } + + let llvm_static_stdcpp = tracked_env_var_os("LLVM_STATIC_STDCPP"); + let llvm_use_libcxx = tracked_env_var_os("LLVM_USE_LIBCXX"); + + let stdcppname = if target.contains("openbsd") { + if target.contains("sparc64") { "estdc++" } else { "c++" } + } else if target.contains("darwin") + || target.contains("freebsd") + || target.contains("windows-gnullvm") + { + "c++" + } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() { + // NetBSD uses a separate library when relocation is required + "stdc++_pic" + } else if llvm_use_libcxx.is_some() { + "c++" + } else { + "stdc++" + }; + + // RISC-V GCC erroneously requires libatomic for sub-word + // atomic operations. FreeBSD uses Clang as its system + // compiler and provides no libatomic in its base system so + // does not want this. + if !target.contains("freebsd") && target.starts_with("riscv") { + println!("cargo:rustc-link-lib=atomic"); + } + + // C++ runtime library + if !target.contains("msvc") { + if let Some(s) = llvm_static_stdcpp { + assert!(!cxxflags.contains("stdlib=libc++")); + let path = PathBuf::from(s); + println!("cargo:rustc-link-search=native={}", path.parent().unwrap().display()); + if target.contains("windows") { + println!("cargo:rustc-link-lib=static:-bundle={}", stdcppname); + } else { + println!("cargo:rustc-link-lib=static={}", stdcppname); + } + } else if cxxflags.contains("stdlib=libc++") { + println!("cargo:rustc-link-lib=c++"); + } else { + println!("cargo:rustc-link-lib={}", stdcppname); + } + } + + // Libstdc++ depends on pthread which Rust doesn't link on MinGW + // since nothing else requires it. + if target.ends_with("windows-gnu") { + println!("cargo:rustc-link-lib=static:-bundle=pthread"); + } +} diff --git a/compiler/rustc_llvm/llvm-wrapper/.editorconfig b/compiler/rustc_llvm/llvm-wrapper/.editorconfig new file mode 100644 index 000000000..865cd45f7 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/.editorconfig @@ -0,0 +1,6 @@ +[*.{h,cpp}] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 2 diff --git a/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp new file mode 100644 index 000000000..97541e615 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/ArchiveWrapper.cpp @@ -0,0 +1,222 @@ +#include "LLVMWrapper.h" + +#include "llvm/Object/Archive.h" +#include "llvm/Object/ArchiveWriter.h" +#include "llvm/Support/Path.h" + +using namespace llvm; +using namespace llvm::object; + +struct RustArchiveMember { + const char *Filename; + const char *Name; + Archive::Child Child; + + RustArchiveMember() + : Filename(nullptr), Name(nullptr), + Child(nullptr, nullptr, nullptr) + { + } + ~RustArchiveMember() {} +}; + +struct RustArchiveIterator { + bool First; + Archive::child_iterator Cur; + Archive::child_iterator End; + std::unique_ptr<Error> Err; + + RustArchiveIterator(Archive::child_iterator Cur, Archive::child_iterator End, + std::unique_ptr<Error> Err) + : First(true), + Cur(Cur), + End(End), + Err(std::move(Err)) {} +}; + +enum class LLVMRustArchiveKind { + GNU, + BSD, + DARWIN, + COFF, +}; + +static Archive::Kind fromRust(LLVMRustArchiveKind Kind) { + switch (Kind) { + case LLVMRustArchiveKind::GNU: + return Archive::K_GNU; + case LLVMRustArchiveKind::BSD: + return Archive::K_BSD; + case LLVMRustArchiveKind::DARWIN: + return Archive::K_DARWIN; + case LLVMRustArchiveKind::COFF: + return Archive::K_COFF; + default: + report_fatal_error("Bad ArchiveKind."); + } +} + +typedef OwningBinary<Archive> *LLVMRustArchiveRef; +typedef RustArchiveMember *LLVMRustArchiveMemberRef; +typedef Archive::Child *LLVMRustArchiveChildRef; +typedef Archive::Child const *LLVMRustArchiveChildConstRef; +typedef RustArchiveIterator *LLVMRustArchiveIteratorRef; + +extern "C" LLVMRustArchiveRef LLVMRustOpenArchive(char *Path) { + ErrorOr<std::unique_ptr<MemoryBuffer>> BufOr = + MemoryBuffer::getFile(Path, -1, false); + if (!BufOr) { + LLVMRustSetLastError(BufOr.getError().message().c_str()); + return nullptr; + } + + Expected<std::unique_ptr<Archive>> ArchiveOr = + Archive::create(BufOr.get()->getMemBufferRef()); + + if (!ArchiveOr) { + LLVMRustSetLastError(toString(ArchiveOr.takeError()).c_str()); + return nullptr; + } + + OwningBinary<Archive> *Ret = new OwningBinary<Archive>( + std::move(ArchiveOr.get()), std::move(BufOr.get())); + + return Ret; +} + +extern "C" void LLVMRustDestroyArchive(LLVMRustArchiveRef RustArchive) { + delete RustArchive; +} + +extern "C" LLVMRustArchiveIteratorRef +LLVMRustArchiveIteratorNew(LLVMRustArchiveRef RustArchive) { + Archive *Archive = RustArchive->getBinary(); + std::unique_ptr<Error> Err = std::make_unique<Error>(Error::success()); + auto Cur = Archive->child_begin(*Err); + if (*Err) { + LLVMRustSetLastError(toString(std::move(*Err)).c_str()); + return nullptr; + } + auto End = Archive->child_end(); + return new RustArchiveIterator(Cur, End, std::move(Err)); +} + +extern "C" LLVMRustArchiveChildConstRef +LLVMRustArchiveIteratorNext(LLVMRustArchiveIteratorRef RAI) { + if (RAI->Cur == RAI->End) + return nullptr; + + // Advancing the iterator validates the next child, and this can + // uncover an error. LLVM requires that we check all Errors, + // so we only advance the iterator if we actually need to fetch + // the next child. + // This means we must not advance the iterator in the *first* call, + // but instead advance it *before* fetching the child in all later calls. + if (!RAI->First) { + ++RAI->Cur; + if (*RAI->Err) { + LLVMRustSetLastError(toString(std::move(*RAI->Err)).c_str()); + return nullptr; + } + } else { + RAI->First = false; + } + + if (RAI->Cur == RAI->End) + return nullptr; + + const Archive::Child &Child = *RAI->Cur.operator->(); + Archive::Child *Ret = new Archive::Child(Child); + + return Ret; +} + +extern "C" void LLVMRustArchiveChildFree(LLVMRustArchiveChildRef Child) { + delete Child; +} + +extern "C" void LLVMRustArchiveIteratorFree(LLVMRustArchiveIteratorRef RAI) { + delete RAI; +} + +extern "C" const char * +LLVMRustArchiveChildName(LLVMRustArchiveChildConstRef Child, size_t *Size) { + Expected<StringRef> NameOrErr = Child->getName(); + if (!NameOrErr) { + // rustc_codegen_llvm currently doesn't use this error string, but it might be + // useful in the future, and in the mean time this tells LLVM that the + // error was not ignored and that it shouldn't abort the process. + LLVMRustSetLastError(toString(NameOrErr.takeError()).c_str()); + return nullptr; + } + StringRef Name = NameOrErr.get(); + *Size = Name.size(); + return Name.data(); +} + +extern "C" const char *LLVMRustArchiveChildData(LLVMRustArchiveChildRef Child, + size_t *Size) { + StringRef Buf; + Expected<StringRef> BufOrErr = Child->getBuffer(); + if (!BufOrErr) { + LLVMRustSetLastError(toString(BufOrErr.takeError()).c_str()); + return nullptr; + } + Buf = BufOrErr.get(); + *Size = Buf.size(); + return Buf.data(); +} + +extern "C" LLVMRustArchiveMemberRef +LLVMRustArchiveMemberNew(char *Filename, char *Name, + LLVMRustArchiveChildRef Child) { + RustArchiveMember *Member = new RustArchiveMember; + Member->Filename = Filename; + Member->Name = Name; + if (Child) + Member->Child = *Child; + return Member; +} + +extern "C" void LLVMRustArchiveMemberFree(LLVMRustArchiveMemberRef Member) { + delete Member; +} + +extern "C" LLVMRustResult +LLVMRustWriteArchive(char *Dst, size_t NumMembers, + const LLVMRustArchiveMemberRef *NewMembers, + bool WriteSymbtab, LLVMRustArchiveKind RustKind) { + + std::vector<NewArchiveMember> Members; + auto Kind = fromRust(RustKind); + + for (size_t I = 0; I < NumMembers; I++) { + auto Member = NewMembers[I]; + assert(Member->Name); + if (Member->Filename) { + Expected<NewArchiveMember> MOrErr = + NewArchiveMember::getFile(Member->Filename, true); + if (!MOrErr) { + LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); + return LLVMRustResult::Failure; + } + MOrErr->MemberName = sys::path::filename(MOrErr->MemberName); + Members.push_back(std::move(*MOrErr)); + } else { + Expected<NewArchiveMember> MOrErr = + NewArchiveMember::getOldMember(Member->Child, true); + if (!MOrErr) { + LLVMRustSetLastError(toString(MOrErr.takeError()).c_str()); + return LLVMRustResult::Failure; + } + Members.push_back(std::move(*MOrErr)); + } + } + + auto Result = writeArchive(Dst, Members, WriteSymbtab, Kind, true, false); + if (!Result) + return LLVMRustResult::Success; + LLVMRustSetLastError(toString(std::move(Result)).c_str()); + + return LLVMRustResult::Failure; +} diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp new file mode 100644 index 000000000..154f554d6 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -0,0 +1,117 @@ +#include "LLVMWrapper.h" +#include "llvm/ProfileData/Coverage/CoverageMapping.h" +#include "llvm/ProfileData/Coverage/CoverageMappingWriter.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/ADT/ArrayRef.h" + +#include <iostream> + +using namespace llvm; + +struct LLVMRustCounterMappingRegion { + coverage::Counter Count; + coverage::Counter FalseCount; + uint32_t FileID; + uint32_t ExpandedFileID; + uint32_t LineStart; + uint32_t ColumnStart; + uint32_t LineEnd; + uint32_t ColumnEnd; + coverage::CounterMappingRegion::RegionKind Kind; +}; + +extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( + const char* const Filenames[], + size_t FilenamesLen, + RustStringRef BufferOut) { +#if LLVM_VERSION_GE(13,0) + SmallVector<std::string,32> FilenameRefs; + for (size_t i = 0; i < FilenamesLen; i++) { + FilenameRefs.push_back(std::string(Filenames[i])); + } +#else + SmallVector<StringRef,32> FilenameRefs; + for (size_t i = 0; i < FilenamesLen; i++) { + FilenameRefs.push_back(StringRef(Filenames[i])); + } +#endif + auto FilenamesWriter = coverage::CoverageFilenamesSectionWriter( + makeArrayRef(FilenameRefs)); + RawRustStringOstream OS(BufferOut); + FilenamesWriter.write(OS); +} + +extern "C" void LLVMRustCoverageWriteMappingToBuffer( + const unsigned *VirtualFileMappingIDs, + unsigned NumVirtualFileMappingIDs, + const coverage::CounterExpression *Expressions, + unsigned NumExpressions, + LLVMRustCounterMappingRegion *RustMappingRegions, + unsigned NumMappingRegions, + RustStringRef BufferOut) { + // Convert from FFI representation to LLVM representation. + SmallVector<coverage::CounterMappingRegion, 0> MappingRegions; + MappingRegions.reserve(NumMappingRegions); + for (const auto &Region : makeArrayRef(RustMappingRegions, NumMappingRegions)) { + MappingRegions.emplace_back( + Region.Count, Region.FalseCount, Region.FileID, Region.ExpandedFileID, + Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd, + Region.Kind); + } + auto CoverageMappingWriter = coverage::CoverageMappingWriter( + makeArrayRef(VirtualFileMappingIDs, NumVirtualFileMappingIDs), + makeArrayRef(Expressions, NumExpressions), + MappingRegions); + RawRustStringOstream OS(BufferOut); + CoverageMappingWriter.write(OS); +} + +extern "C" LLVMValueRef LLVMRustCoverageCreatePGOFuncNameVar(LLVMValueRef F, const char *FuncName) { + StringRef FuncNameRef(FuncName); + return wrap(createPGOFuncNameVar(*cast<Function>(unwrap(F)), FuncNameRef)); +} + +extern "C" uint64_t LLVMRustCoverageHashCString(const char *StrVal) { + StringRef StrRef(StrVal); + return IndexedInstrProf::ComputeHash(StrRef); +} + +extern "C" uint64_t LLVMRustCoverageHashByteArray( + const char *Bytes, + unsigned NumBytes) { + StringRef StrRef(Bytes, NumBytes); + return IndexedInstrProf::ComputeHash(StrRef); +} + +static void WriteSectionNameToString(LLVMModuleRef M, + InstrProfSectKind SK, + RustStringRef Str) { + Triple TargetTriple(unwrap(M)->getTargetTriple()); + auto name = getInstrProfSectionName(SK, TargetTriple.getObjectFormat()); + RawRustStringOstream OS(Str); + OS << name; +} + +extern "C" void LLVMRustCoverageWriteMapSectionNameToString(LLVMModuleRef M, + RustStringRef Str) { + WriteSectionNameToString(M, IPSK_covmap, Str); +} + +extern "C" void LLVMRustCoverageWriteFuncSectionNameToString(LLVMModuleRef M, + RustStringRef Str) { + WriteSectionNameToString(M, IPSK_covfun, Str); +} + +extern "C" void LLVMRustCoverageWriteMappingVarNameToString(RustStringRef Str) { + auto name = getCoverageMappingVarName(); + RawRustStringOstream OS(Str); + OS << name; +} + +extern "C" uint32_t LLVMRustCoverageMappingVersion() { +#if LLVM_VERSION_GE(13, 0) + return coverage::CovMapVersion::Version6; +#else + return coverage::CovMapVersion::Version5; +#endif +} diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h new file mode 100644 index 000000000..015c1c52b --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h @@ -0,0 +1,121 @@ +#include "llvm-c/BitReader.h" +#include "llvm-c/Core.h" +#include "llvm-c/Object.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Triple.h" +#include "llvm/Analysis/Lint.h" +#include "llvm/Analysis/Passes.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/Memory.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Vectorize.h" + +#define LLVM_VERSION_GE(major, minor) \ + (LLVM_VERSION_MAJOR > (major) || \ + LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR >= (minor)) + +#define LLVM_VERSION_LT(major, minor) (!LLVM_VERSION_GE((major), (minor))) + +#include "llvm/IR/LegacyPassManager.h" + +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" + +#include "llvm/IR/DIBuilder.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/IRPrintingPasses.h" +#include "llvm/Linker/Linker.h" + +extern "C" void LLVMRustSetLastError(const char *); + +enum class LLVMRustResult { Success, Failure }; + +enum LLVMRustAttribute { + AlwaysInline = 0, + ByVal = 1, + Cold = 2, + InlineHint = 3, + MinSize = 4, + Naked = 5, + NoAlias = 6, + NoCapture = 7, + NoInline = 8, + NonNull = 9, + NoRedZone = 10, + NoReturn = 11, + NoUnwind = 12, + OptimizeForSize = 13, + ReadOnly = 14, + SExt = 15, + StructRet = 16, + UWTable = 17, + ZExt = 18, + InReg = 19, + SanitizeThread = 20, + SanitizeAddress = 21, + SanitizeMemory = 22, + NonLazyBind = 23, + OptimizeNone = 24, + ReturnsTwice = 25, + ReadNone = 26, + InaccessibleMemOnly = 27, + SanitizeHWAddress = 28, + WillReturn = 29, + StackProtectReq = 30, + StackProtectStrong = 31, + StackProtect = 32, + NoUndef = 33, + SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, +#if LLVM_VERSION_GE(15, 0) + AllocatedPointer = 38, + AllocAlign = 39, +#endif +}; + +typedef struct OpaqueRustString *RustStringRef; +typedef struct LLVMOpaqueTwine *LLVMTwineRef; +typedef struct LLVMOpaqueSMDiagnostic *LLVMSMDiagnosticRef; + +extern "C" void LLVMRustStringWriteImpl(RustStringRef Str, const char *Ptr, + size_t Size); + +class RawRustStringOstream : public llvm::raw_ostream { + RustStringRef Str; + uint64_t Pos; + + void write_impl(const char *Ptr, size_t Size) override { + LLVMRustStringWriteImpl(Str, Ptr, Size); + Pos += Size; + } + + uint64_t current_pos() const override { return Pos; } + +public: + explicit RawRustStringOstream(RustStringRef Str) : Str(Str), Pos(0) {} + + ~RawRustStringOstream() { + // LLVM requires this. + flush(); + } +}; diff --git a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp new file mode 100644 index 000000000..8766e96f0 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp @@ -0,0 +1,48 @@ +#include "llvm/Linker/Linker.h" + +#include "LLVMWrapper.h" + +using namespace llvm; + +struct RustLinker { + Linker L; + LLVMContext &Ctx; + + RustLinker(Module &M) : + L(M), + Ctx(M.getContext()) + {} +}; + +extern "C" RustLinker* +LLVMRustLinkerNew(LLVMModuleRef DstRef) { + Module *Dst = unwrap(DstRef); + + return new RustLinker(*Dst); +} + +extern "C" void +LLVMRustLinkerFree(RustLinker *L) { + delete L; +} + +extern "C" bool +LLVMRustLinkerAdd(RustLinker *L, char *BC, size_t Len) { + std::unique_ptr<MemoryBuffer> Buf = + MemoryBuffer::getMemBufferCopy(StringRef(BC, Len)); + + Expected<std::unique_ptr<Module>> SrcOrError = + llvm::getLazyBitcodeModule(Buf->getMemBufferRef(), L->Ctx); + if (!SrcOrError) { + LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); + return false; + } + + auto Src = std::move(*SrcOrError); + + if (L->L.linkInModule(std::move(Src))) { + LLVMRustSetLastError(""); + return false; + } + return true; +} diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp new file mode 100644 index 000000000..0a6bd4999 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -0,0 +1,1763 @@ +#include <stdio.h> + +#include <vector> +#include <set> + +#include "LLVMWrapper.h" + +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/InitializePasses.h" +#include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/AssemblyAnnotationWriter.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/IRObjectFile.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Host.h" +#if LLVM_VERSION_LT(14, 0) +#include "llvm/Support/TargetRegistry.h" +#else +#include "llvm/MC/TargetRegistry.h" +#endif +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/Transforms/IPO/FunctionImport.h" +#include "llvm/Transforms/Utils/AddDiscriminators.h" +#include "llvm/Transforms/Utils/FunctionImportUtils.h" +#include "llvm/LTO/LTO.h" +#include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm-c/Transforms/PassManagerBuilder.h" + +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" +#include "llvm/Support/TimeProfiler.h" +#include "llvm/Transforms/Instrumentation/GCOVProfiler.h" +#include "llvm/Transforms/Instrumentation/InstrProfiling.h" +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" +#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" +#include "llvm/Transforms/Utils/CanonicalizeAliases.h" +#include "llvm/Transforms/Utils/NameAnonGlobals.h" +#include "llvm/Transforms/Utils.h" + +using namespace llvm; + +typedef struct LLVMOpaquePass *LLVMPassRef; +typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; + +DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) +DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +extern "C" void LLVMInitializePasses() { + PassRegistry &Registry = *PassRegistry::getPassRegistry(); + initializeCore(Registry); + initializeCodeGen(Registry); + initializeScalarOpts(Registry); + initializeVectorization(Registry); + initializeIPO(Registry); + initializeAnalysis(Registry); + initializeTransformUtils(Registry); + initializeInstCombine(Registry); + initializeInstrumentation(Registry); + initializeTarget(Registry); +} + +extern "C" void LLVMTimeTraceProfilerInitialize() { + timeTraceProfilerInitialize( + /* TimeTraceGranularity */ 0, + /* ProcName */ "rustc"); +} + +extern "C" void LLVMTimeTraceProfilerFinishThread() { + timeTraceProfilerFinishThread(); +} + +extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) { + StringRef FN(FileName); + std::error_code EC; + raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways); + + timeTraceProfilerWrite(OS); + timeTraceProfilerCleanup(); +} + +extern "C" LLVMPassRef LLVMRustFindAndCreatePass(const char *PassName) { +#if LLVM_VERSION_LT(15, 0) + StringRef SR(PassName); + PassRegistry *PR = PassRegistry::getPassRegistry(); + + const PassInfo *PI = PR->getPassInfo(SR); + if (PI) { + return wrap(PI->createPass()); + } + return nullptr; +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateAddressSanitizerFunctionPass(bool Recover) { +#if LLVM_VERSION_LT(15, 0) + const bool CompileKernel = false; + const bool UseAfterScope = true; + + return wrap(createAddressSanitizerFunctionPass(CompileKernel, Recover, UseAfterScope)); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateModuleAddressSanitizerPass(bool Recover) { +#if LLVM_VERSION_LT(15, 0) + const bool CompileKernel = false; + + return wrap(createModuleAddressSanitizerLegacyPassPass(CompileKernel, Recover)); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateMemorySanitizerPass(int TrackOrigins, bool Recover) { +#if LLVM_VERSION_LT(15, 0) + const bool CompileKernel = false; + + return wrap(createMemorySanitizerLegacyPassPass( + MemorySanitizerOptions{TrackOrigins, Recover, CompileKernel})); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateThreadSanitizerPass() { +#if LLVM_VERSION_LT(15, 0) + return wrap(createThreadSanitizerLegacyPassPass()); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) { +#if LLVM_VERSION_LT(15, 0) + const bool CompileKernel = false; + + return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover)); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustAddPass(LLVMPassManagerRef PMR, LLVMPassRef RustPass) { +#if LLVM_VERSION_LT(15, 0) + assert(RustPass); + Pass *Pass = unwrap(RustPass); + PassManagerBase *PMB = unwrap(PMR); + PMB->add(Pass); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" LLVMPassManagerBuilderRef LLVMRustPassManagerBuilderCreate() { +#if LLVM_VERSION_LT(15, 0) + return LLVMPassManagerBuilderCreate(); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderDispose(LLVMPassManagerBuilderRef PMB) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderDispose(PMB); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderPopulateFunctionPassManager( + LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderPopulateFunctionPassManager(PMB, PM); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderPopulateModulePassManager( + LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderPopulateModulePassManager(PMB, PM); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderPopulateLTOPassManager( + LLVMPassManagerBuilderRef PMB, LLVMPassManagerRef PM, bool Internalize, bool RunInliner) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderPopulateLTOPassManager(PMB, PM, Internalize, RunInliner); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" +void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( + LLVMPassManagerBuilderRef PMBR, + LLVMPassManagerRef PMR +) { +#if LLVM_VERSION_LT(15, 0) + unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" void LLVMRustPassManagerBuilderUseInlinerWithThreshold( + LLVMPassManagerBuilderRef PMB, unsigned Threshold) { +#if LLVM_VERSION_LT(15, 0) + LLVMPassManagerBuilderUseInlinerWithThreshold(PMB, Threshold); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +extern "C" +void LLVMRustAddLastExtensionPasses( + LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { +#if LLVM_VERSION_LT(15, 0) + auto AddExtensionPasses = [Passes, NumPasses]( + const PassManagerBuilder &Builder, PassManagerBase &PM) { + for (size_t I = 0; I < NumPasses; I++) { + PM.add(unwrap(Passes[I])); + } + }; + // Add the passes to both of the pre-finalization extension points, + // so they are run for optimized and non-optimized builds. + unwrap(PMBR)->addExtension(PassManagerBuilder::EP_OptimizerLast, + AddExtensionPasses); + unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, + AddExtensionPasses); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +#ifdef LLVM_COMPONENT_X86 +#define SUBTARGET_X86 SUBTARGET(X86) +#else +#define SUBTARGET_X86 +#endif + +#ifdef LLVM_COMPONENT_ARM +#define SUBTARGET_ARM SUBTARGET(ARM) +#else +#define SUBTARGET_ARM +#endif + +#ifdef LLVM_COMPONENT_AARCH64 +#define SUBTARGET_AARCH64 SUBTARGET(AArch64) +#else +#define SUBTARGET_AARCH64 +#endif + +#ifdef LLVM_COMPONENT_AVR +#define SUBTARGET_AVR SUBTARGET(AVR) +#else +#define SUBTARGET_AVR +#endif + +#ifdef LLVM_COMPONENT_M68k +#define SUBTARGET_M68K SUBTARGET(M68k) +#else +#define SUBTARGET_M68K +#endif + +#ifdef LLVM_COMPONENT_MIPS +#define SUBTARGET_MIPS SUBTARGET(Mips) +#else +#define SUBTARGET_MIPS +#endif + +#ifdef LLVM_COMPONENT_POWERPC +#define SUBTARGET_PPC SUBTARGET(PPC) +#else +#define SUBTARGET_PPC +#endif + +#ifdef LLVM_COMPONENT_SYSTEMZ +#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) +#else +#define SUBTARGET_SYSTEMZ +#endif + +#ifdef LLVM_COMPONENT_MSP430 +#define SUBTARGET_MSP430 SUBTARGET(MSP430) +#else +#define SUBTARGET_MSP430 +#endif + +#ifdef LLVM_COMPONENT_RISCV +#define SUBTARGET_RISCV SUBTARGET(RISCV) +#else +#define SUBTARGET_RISCV +#endif + +#ifdef LLVM_COMPONENT_SPARC +#define SUBTARGET_SPARC SUBTARGET(Sparc) +#else +#define SUBTARGET_SPARC +#endif + +#ifdef LLVM_COMPONENT_HEXAGON +#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) +#else +#define SUBTARGET_HEXAGON +#endif + +#define GEN_SUBTARGETS \ + SUBTARGET_X86 \ + SUBTARGET_ARM \ + SUBTARGET_AARCH64 \ + SUBTARGET_AVR \ + SUBTARGET_M68K \ + SUBTARGET_MIPS \ + SUBTARGET_PPC \ + SUBTARGET_SYSTEMZ \ + SUBTARGET_MSP430 \ + SUBTARGET_SPARC \ + SUBTARGET_HEXAGON \ + SUBTARGET_RISCV \ + +#define SUBTARGET(x) \ + namespace llvm { \ + extern const SubtargetFeatureKV x##FeatureKV[]; \ + extern const SubtargetFeatureKV x##SubTypeKV[]; \ + } + +GEN_SUBTARGETS +#undef SUBTARGET + +extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, + const char *Feature) { + TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + return MCInfo->checkFeatures(std::string("+") + Feature); +} + +enum class LLVMRustCodeModel { + Tiny, + Small, + Kernel, + Medium, + Large, + None, +}; + +static Optional<CodeModel::Model> fromRust(LLVMRustCodeModel Model) { + switch (Model) { + case LLVMRustCodeModel::Tiny: + return CodeModel::Tiny; + case LLVMRustCodeModel::Small: + return CodeModel::Small; + case LLVMRustCodeModel::Kernel: + return CodeModel::Kernel; + case LLVMRustCodeModel::Medium: + return CodeModel::Medium; + case LLVMRustCodeModel::Large: + return CodeModel::Large; + case LLVMRustCodeModel::None: + return None; + default: + report_fatal_error("Bad CodeModel."); + } +} + +enum class LLVMRustCodeGenOptLevel { + None, + Less, + Default, + Aggressive, +}; + +static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) { + switch (Level) { + case LLVMRustCodeGenOptLevel::None: + return CodeGenOpt::None; + case LLVMRustCodeGenOptLevel::Less: + return CodeGenOpt::Less; + case LLVMRustCodeGenOptLevel::Default: + return CodeGenOpt::Default; + case LLVMRustCodeGenOptLevel::Aggressive: + return CodeGenOpt::Aggressive; + default: + report_fatal_error("Bad CodeGenOptLevel."); + } +} + +enum class LLVMRustPassBuilderOptLevel { + O0, + O1, + O2, + O3, + Os, + Oz, +}; + +#if LLVM_VERSION_LT(14,0) +using OptimizationLevel = PassBuilder::OptimizationLevel; +#endif + +static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { + switch (Level) { + case LLVMRustPassBuilderOptLevel::O0: + return OptimizationLevel::O0; + case LLVMRustPassBuilderOptLevel::O1: + return OptimizationLevel::O1; + case LLVMRustPassBuilderOptLevel::O2: + return OptimizationLevel::O2; + case LLVMRustPassBuilderOptLevel::O3: + return OptimizationLevel::O3; + case LLVMRustPassBuilderOptLevel::Os: + return OptimizationLevel::Os; + case LLVMRustPassBuilderOptLevel::Oz: + return OptimizationLevel::Oz; + default: + report_fatal_error("Bad PassBuilderOptLevel."); + } +} + +enum class LLVMRustRelocModel { + Static, + PIC, + DynamicNoPic, + ROPI, + RWPI, + ROPIRWPI, +}; + +static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { + switch (RustReloc) { + case LLVMRustRelocModel::Static: + return Reloc::Static; + case LLVMRustRelocModel::PIC: + return Reloc::PIC_; + case LLVMRustRelocModel::DynamicNoPic: + return Reloc::DynamicNoPIC; + case LLVMRustRelocModel::ROPI: + return Reloc::ROPI; + case LLVMRustRelocModel::RWPI: + return Reloc::RWPI; + case LLVMRustRelocModel::ROPIRWPI: + return Reloc::ROPI_RWPI; + } + report_fatal_error("Bad RelocModel."); +} + +#ifdef LLVM_RUSTLLVM +/// getLongestEntryLength - Return the length of the longest entry in the table. +template<typename KV> +static size_t getLongestEntryLength(ArrayRef<KV> Table) { + size_t MaxLen = 0; + for (auto &I : Table) + MaxLen = std::max(MaxLen, std::strlen(I.Key)); + return MaxLen; +} + +extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM) { + const TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + const Triple::ArchType HostArch = Triple(sys::getProcessTriple()).getArch(); + const Triple::ArchType TargetArch = Target->getTargetTriple().getArch(); + const ArrayRef<SubtargetSubTypeKV> CPUTable = MCInfo->getCPUTable(); + unsigned MaxCPULen = getLongestEntryLength(CPUTable); + + printf("Available CPUs for this target:\n"); + if (HostArch == TargetArch) { + const StringRef HostCPU = sys::getHostCPUName(); + printf(" %-*s - Select the CPU of the current host (currently %.*s).\n", + MaxCPULen, "native", (int)HostCPU.size(), HostCPU.data()); + } + for (auto &CPU : CPUTable) + printf(" %-*s\n", MaxCPULen, CPU.Key); + printf("\n"); +} + +extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { + const TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); + return FeatTable.size(); +} + +extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, + const char** Feature, const char** Desc) { + const TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + const ArrayRef<SubtargetFeatureKV> FeatTable = MCInfo->getFeatureTable(); + const SubtargetFeatureKV Feat = FeatTable[Index]; + *Feature = Feat.Key; + *Desc = Feat.Desc; +} + +#else + +extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef) { + printf("Target CPU help is not supported by this LLVM version.\n\n"); +} + +extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef) { + return 0; +} + +extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef, const char**, const char**) {} +#endif + +extern "C" const char* LLVMRustGetHostCPUName(size_t *len) { + StringRef Name = sys::getHostCPUName(); + *len = Name.size(); + return Name.data(); +} + +extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( + const char *TripleStr, const char *CPU, const char *Feature, + const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, + LLVMRustCodeGenOptLevel RustOptLevel, bool UseSoftFloat, + bool FunctionSections, + bool DataSections, + bool UniqueSectionNames, + bool TrapUnreachable, + bool Singlethread, + bool AsmComments, + bool EmitStackSizeSection, + bool RelaxELFRelocations, + bool UseInitArray, + const char *SplitDwarfFile) { + + auto OptLevel = fromRust(RustOptLevel); + auto RM = fromRust(RustReloc); + auto CM = fromRust(RustCM); + + std::string Error; + Triple Trip(Triple::normalize(TripleStr)); + const llvm::Target *TheTarget = + TargetRegistry::lookupTarget(Trip.getTriple(), Error); + if (TheTarget == nullptr) { + LLVMRustSetLastError(Error.c_str()); + return nullptr; + } + + TargetOptions Options; + + Options.FloatABIType = FloatABI::Default; + if (UseSoftFloat) { + Options.FloatABIType = FloatABI::Soft; + } + Options.DataSections = DataSections; + Options.FunctionSections = FunctionSections; + Options.UniqueSectionNames = UniqueSectionNames; + Options.MCOptions.AsmVerbose = AsmComments; + Options.MCOptions.PreserveAsmComments = AsmComments; + Options.MCOptions.ABIName = ABIStr; + if (SplitDwarfFile) { + Options.MCOptions.SplitDwarfFile = SplitDwarfFile; + } + Options.RelaxELFRelocations = RelaxELFRelocations; + Options.UseInitArray = UseInitArray; + + if (TrapUnreachable) { + // Tell LLVM to codegen `unreachable` into an explicit trap instruction. + // This limits the extent of possible undefined behavior in some cases, as + // it prevents control flow from "falling through" into whatever code + // happens to be laid out next in memory. + Options.TrapUnreachable = true; + } + + if (Singlethread) { + Options.ThreadModel = ThreadModel::Single; + } + + Options.EmitStackSizeSection = EmitStackSizeSection; + + TargetMachine *TM = TheTarget->createTargetMachine( + Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); + return wrap(TM); +} + +extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { + delete unwrap(TM); +} + +extern "C" void LLVMRustConfigurePassManagerBuilder( + LLVMPassManagerBuilderRef PMBR, LLVMRustCodeGenOptLevel OptLevel, + bool MergeFunctions, bool SLPVectorize, bool LoopVectorize, bool PrepareForThinLTO, + const char* PGOGenPath, const char* PGOUsePath, const char* PGOSampleUsePath, + int SizeLevel) { +#if LLVM_VERSION_LT(15, 0) + unwrap(PMBR)->MergeFunctions = MergeFunctions; + unwrap(PMBR)->SLPVectorize = SLPVectorize; + unwrap(PMBR)->OptLevel = fromRust(OptLevel); + unwrap(PMBR)->LoopVectorize = LoopVectorize; + unwrap(PMBR)->PrepareForThinLTO = PrepareForThinLTO; + unwrap(PMBR)->SizeLevel = SizeLevel; + unwrap(PMBR)->DisableUnrollLoops = SizeLevel != 0; + + if (PGOGenPath) { + assert(!PGOUsePath && !PGOSampleUsePath); + unwrap(PMBR)->EnablePGOInstrGen = true; + unwrap(PMBR)->PGOInstrGen = PGOGenPath; + } else if (PGOUsePath) { + assert(!PGOSampleUsePath); + unwrap(PMBR)->PGOInstrUse = PGOUsePath; + } else if (PGOSampleUsePath) { + unwrap(PMBR)->PGOSampleUse = PGOSampleUsePath; + } +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + +// Unfortunately, the LLVM C API doesn't provide a way to set the `LibraryInfo` +// field of a PassManagerBuilder, we expose our own method of doing so. +extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR, + LLVMModuleRef M, + bool DisableSimplifyLibCalls) { + Triple TargetTriple(unwrap(M)->getTargetTriple()); + TargetLibraryInfoImpl *TLI = new TargetLibraryInfoImpl(TargetTriple); + if (DisableSimplifyLibCalls) + TLI->disableAllFunctions(); + unwrap(PMBR)->LibraryInfo = TLI; +} + +// Unfortunately, the LLVM C API doesn't provide a way to create the +// TargetLibraryInfo pass, so we use this method to do so. +extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, + bool DisableSimplifyLibCalls) { + Triple TargetTriple(unwrap(M)->getTargetTriple()); + TargetLibraryInfoImpl TLII(TargetTriple); + if (DisableSimplifyLibCalls) + TLII.disableAllFunctions(); + unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); +} + +// Unfortunately, the LLVM C API doesn't provide an easy way of iterating over +// all the functions in a module, so we do that manually here. You'll find +// similar code in clang's BackendUtil.cpp file. +extern "C" void LLVMRustRunFunctionPassManager(LLVMPassManagerRef PMR, + LLVMModuleRef M) { + llvm::legacy::FunctionPassManager *P = + unwrap<llvm::legacy::FunctionPassManager>(PMR); + P->doInitialization(); + + // Upgrade all calls to old intrinsics first. + for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E;) + UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + + for (Module::iterator I = unwrap(M)->begin(), E = unwrap(M)->end(); I != E; + ++I) + if (!I->isDeclaration()) + P->run(*I); + + P->doFinalization(); +} + +extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { + // Initializing the command-line options more than once is not allowed. So, + // check if they've already been initialized. (This could happen if we're + // being called from rustpkg, for example). If the arguments change, then + // that's just kinda unfortunate. + static bool Initialized = false; + if (Initialized) + return; + Initialized = true; + cl::ParseCommandLineOptions(Argc, Argv); +} + +enum class LLVMRustFileType { + AssemblyFile, + ObjectFile, +}; + +static CodeGenFileType fromRust(LLVMRustFileType Type) { + switch (Type) { + case LLVMRustFileType::AssemblyFile: + return CGFT_AssemblyFile; + case LLVMRustFileType::ObjectFile: + return CGFT_ObjectFile; + default: + report_fatal_error("Bad FileType."); + } +} + +extern "C" LLVMRustResult +LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, + LLVMModuleRef M, const char *Path, const char *DwoPath, + LLVMRustFileType RustFileType) { + llvm::legacy::PassManager *PM = unwrap<llvm::legacy::PassManager>(PMR); + auto FileType = fromRust(RustFileType); + + std::string ErrorInfo; + std::error_code EC; + raw_fd_ostream OS(Path, EC, sys::fs::OF_None); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + + buffer_ostream BOS(OS); + if (DwoPath) { + raw_fd_ostream DOS(DwoPath, EC, sys::fs::OF_None); + EC.clear(); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + buffer_ostream DBOS(DOS); + unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, false); + PM->run(*unwrap(M)); + } else { + unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, false); + PM->run(*unwrap(M)); + } + + // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output + // stream (OS), so the only real safe place to delete this is here? Don't we + // wish this was written in Rust? + LLVMDisposePassManager(PMR); + return LLVMRustResult::Success; +} + +extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler + const char*, // pass name + const char*); // IR name +extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler + +std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { + if (any_isa<const Module *>(WrappedIr)) + return any_cast<const Module *>(WrappedIr)->getName().str(); + if (any_isa<const Function *>(WrappedIr)) + return any_cast<const Function *>(WrappedIr)->getName().str(); + if (any_isa<const Loop *>(WrappedIr)) + return any_cast<const Loop *>(WrappedIr)->getName().str(); + if (any_isa<const LazyCallGraph::SCC *>(WrappedIr)) + return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName(); + return "<UNKNOWN>"; +} + + +void LLVMSelfProfileInitializeCallbacks( + PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler, + LLVMRustSelfProfileBeforePassCallback BeforePassCallback, + LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { + PIC.registerBeforeNonSkippedPassCallback([LlvmSelfProfiler, BeforePassCallback]( + StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); + + PIC.registerAfterPassCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any IR, + const PreservedAnalyses &Preserved) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerAfterPassInvalidatedCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback]( + StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); + + PIC.registerAfterAnalysisCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { + AfterPassCallback(LlvmSelfProfiler); + }); +} + +enum class LLVMRustOptStage { + PreLinkNoLTO, + PreLinkThinLTO, + PreLinkFatLTO, + ThinLTO, + FatLTO, +}; + +struct LLVMRustSanitizerOptions { + bool SanitizeAddress; + bool SanitizeAddressRecover; + bool SanitizeMemory; + bool SanitizeMemoryRecover; + int SanitizeMemoryTrackOrigins; + bool SanitizeThread; + bool SanitizeHWAddress; + bool SanitizeHWAddressRecover; +}; + +extern "C" LLVMRustResult +LLVMRustOptimizeWithNewPassManager( + LLVMModuleRef ModuleRef, + LLVMTargetMachineRef TMRef, + LLVMRustPassBuilderOptLevel OptLevelRust, + LLVMRustOptStage OptStage, + bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers, + bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, + bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, + LLVMRustSanitizerOptions *SanitizerOptions, + const char *PGOGenPath, const char *PGOUsePath, + bool InstrumentCoverage, bool InstrumentGCOV, + const char *PGOSampleUsePath, bool DebugInfoForProfiling, + void* LlvmSelfProfiler, + LLVMRustSelfProfileBeforePassCallback BeforePassCallback, + LLVMRustSelfProfileAfterPassCallback AfterPassCallback, + const char *ExtraPasses, size_t ExtraPassesLen, + const char *LLVMPlugins, size_t LLVMPluginsLen) { + Module *TheModule = unwrap(ModuleRef); + TargetMachine *TM = unwrap(TMRef); + OptimizationLevel OptLevel = fromRust(OptLevelRust); + + + PipelineTuningOptions PTO; + PTO.LoopUnrolling = UnrollLoops; + PTO.LoopInterleaving = UnrollLoops; + PTO.LoopVectorization = LoopVectorize; + PTO.SLPVectorization = SLPVectorize; + PTO.MergeFunctions = MergeFunctions; + + // FIXME: We may want to expose this as an option. + bool DebugPassManager = false; + + PassInstrumentationCallbacks PIC; + StandardInstrumentations SI(DebugPassManager); + SI.registerCallbacks(PIC); + + if (LlvmSelfProfiler){ + LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback); + } + + Optional<PGOOptions> PGOOpt; + if (PGOGenPath) { + assert(!PGOUsePath && !PGOSampleUsePath); + PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr, + PGOOptions::NoCSAction, DebugInfoForProfiling); + } else if (PGOUsePath) { + assert(!PGOSampleUsePath); + PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse, + PGOOptions::NoCSAction, DebugInfoForProfiling); + } else if (PGOSampleUsePath) { + PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse, + PGOOptions::NoCSAction, DebugInfoForProfiling); + } else if (DebugInfoForProfiling) { + PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction, + PGOOptions::NoCSAction, DebugInfoForProfiling); + } + +#if LLVM_VERSION_GE(13, 0) + PassBuilder PB(TM, PTO, PGOOpt, &PIC); + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; +#else + PassBuilder PB(DebugPassManager, TM, PTO, PGOOpt, &PIC); + LoopAnalysisManager LAM(DebugPassManager); + FunctionAnalysisManager FAM(DebugPassManager); + CGSCCAnalysisManager CGAM(DebugPassManager); + ModuleAnalysisManager MAM(DebugPassManager); +#endif + + FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); + + Triple TargetTriple(TheModule->getTargetTriple()); + std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple)); + if (DisableSimplifyLibCalls) + TLII->disableAllFunctions(); + FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); + + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + // We manually collect pipeline callbacks so we can apply them at O0, where the + // PassBuilder does not create a pipeline. + std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> + PipelineStartEPCallbacks; + std::vector<std::function<void(ModulePassManager &, OptimizationLevel)>> + OptimizerLastEPCallbacks; + + if (VerifyIR) { + PipelineStartEPCallbacks.push_back( + [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(VerifierPass()); + } + ); + } + + if (InstrumentGCOV) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(GCOVProfilerPass(GCOVOptions::getDefault())); + } + ); + } + + if (InstrumentCoverage) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + InstrProfOptions Options; + MPM.addPass(InstrProfiling(Options, false)); + } + ); + } + + if (SanitizerOptions) { + if (SanitizerOptions->SanitizeMemory) { + MemorySanitizerOptions Options( + SanitizerOptions->SanitizeMemoryTrackOrigins, + SanitizerOptions->SanitizeMemoryRecover, + /*CompileKernel=*/false); + OptimizerLastEPCallbacks.push_back( + [Options](ModulePassManager &MPM, OptimizationLevel Level) { +#if LLVM_VERSION_GE(14, 0) + MPM.addPass(ModuleMemorySanitizerPass(Options)); +#else + MPM.addPass(MemorySanitizerPass(Options)); +#endif + MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options))); + } + ); + } + + if (SanitizerOptions->SanitizeThread) { + OptimizerLastEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { +#if LLVM_VERSION_GE(14, 0) + MPM.addPass(ModuleThreadSanitizerPass()); +#else + MPM.addPass(ThreadSanitizerPass()); +#endif + MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); + } + ); + } + + if (SanitizerOptions->SanitizeAddress) { + OptimizerLastEPCallbacks.push_back( + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +#if LLVM_VERSION_LT(15, 0) + MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>()); +#endif +#if LLVM_VERSION_GE(14, 0) + AddressSanitizerOptions opts = AddressSanitizerOptions{ + /*CompileKernel=*/false, + SanitizerOptions->SanitizeAddressRecover, + /*UseAfterScope=*/true, + AsanDetectStackUseAfterReturnMode::Runtime, + }; + MPM.addPass(ModuleAddressSanitizerPass(opts)); +#else + MPM.addPass(ModuleAddressSanitizerPass( + /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover)); + MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( + /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover, + /*UseAfterScope=*/true))); +#endif + } + ); + } + if (SanitizerOptions->SanitizeHWAddress) { + OptimizerLastEPCallbacks.push_back( + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +#if LLVM_VERSION_GE(14, 0) + HWAddressSanitizerOptions opts( + /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover, + /*DisableOptimization=*/false); + MPM.addPass(HWAddressSanitizerPass(opts)); +#else + MPM.addPass(HWAddressSanitizerPass( + /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover)); +#endif + } + ); + } + } + + if (LLVMPluginsLen) { + auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); + SmallVector<StringRef> Plugins; + PluginsStr.split(Plugins, ',', -1, false); + for (auto PluginPath: Plugins) { + auto Plugin = PassPlugin::Load(PluginPath.str()); + if (!Plugin) { + LLVMRustSetLastError(("Failed to load pass plugin" + PluginPath.str()).c_str()); + continue; + } + Plugin->registerPassBuilderCallbacks(PB); + } + } + +#if LLVM_VERSION_GE(13, 0) + ModulePassManager MPM; +#else + ModulePassManager MPM(DebugPassManager); +#endif + bool NeedThinLTOBufferPasses = UseThinLTOBuffers; + if (!NoPrepopulatePasses) { + // The pre-link pipelines don't support O0 and require using budilO0DefaultPipeline() instead. + // At the same time, the LTO pipelines do support O0 and using them is required. + bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; + if (OptLevel == OptimizationLevel::O0 && !IsLTO) { + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); + + // Pass false as we manually schedule ThinLTOBufferPasses below. + MPM = PB.buildO0DefaultPipeline(OptLevel, /* PreLinkLTO */ false); + } else { + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback(C); + if (OptStage != LLVMRustOptStage::PreLinkThinLTO) { + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); + } + + switch (OptStage) { + case LLVMRustOptStage::PreLinkNoLTO: + MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager); + break; + case LLVMRustOptStage::PreLinkThinLTO: + MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); + // The ThinLTOPreLink pipeline already includes ThinLTOBuffer passes. However, callback + // passes may still run afterwards. This means we need to run the buffer passes again. + // FIXME: In LLVM 13, the ThinLTOPreLink pipeline also runs OptimizerLastEPCallbacks + // before the RequiredLTOPreLinkPasses, in which case we can remove these hacks. + if (OptimizerLastEPCallbacks.empty()) + NeedThinLTOBufferPasses = false; + for (const auto &C : OptimizerLastEPCallbacks) + C(MPM, OptLevel); + break; + case LLVMRustOptStage::PreLinkFatLTO: + MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); + NeedThinLTOBufferPasses = false; + break; + case LLVMRustOptStage::ThinLTO: + // FIXME: Does it make sense to pass the ModuleSummaryIndex? + // It only seems to be needed for C++ specific optimizations. + MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); + break; + case LLVMRustOptStage::FatLTO: + MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); + break; + } + } + } else { + // We're not building any of the default pipelines but we still want to + // add the verifier, instrumentation, etc passes if they were requested + for (const auto &C : PipelineStartEPCallbacks) + C(MPM, OptLevel); + for (const auto &C : OptimizerLastEPCallbacks) + C(MPM, OptLevel); + } + + if (ExtraPassesLen) { + if (auto Err = PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { + std::string ErrMsg = toString(std::move(Err)); + LLVMRustSetLastError(ErrMsg.c_str()); + return LLVMRustResult::Failure; + } + } + + if (NeedThinLTOBufferPasses) { + MPM.addPass(CanonicalizeAliasesPass()); + MPM.addPass(NameAnonGlobalPass()); + } + + // Upgrade all calls to old intrinsics first. + for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) + UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + + MPM.run(*TheModule, MAM); + return LLVMRustResult::Success; +} + +// Callback to demangle function name +// Parameters: +// * name to be demangled +// * name len +// * output buffer +// * output buffer len +// Returns len of demangled string, or 0 if demangle failed. +typedef size_t (*DemangleFn)(const char*, size_t, char*, size_t); + + +namespace { + +class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { + DemangleFn Demangle; + std::vector<char> Buf; + +public: + RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} + + // Return empty string if demangle failed + // or if name does not need to be demangled + StringRef CallDemangle(StringRef name) { + if (!Demangle) { + return StringRef(); + } + + if (Buf.size() < name.size() * 2) { + // Semangled name usually shorter than mangled, + // but allocate twice as much memory just in case + Buf.resize(name.size() * 2); + } + + auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); + if (!R) { + // Demangle failed. + return StringRef(); + } + + auto Demangled = StringRef(Buf.data(), R); + if (Demangled == name) { + // Do not print anything if demangled name is equal to mangled. + return StringRef(); + } + + return Demangled; + } + + void emitFunctionAnnot(const Function *F, + formatted_raw_ostream &OS) override { + StringRef Demangled = CallDemangle(F->getName()); + if (Demangled.empty()) { + return; + } + + OS << "; " << Demangled << "\n"; + } + + void emitInstructionAnnot(const Instruction *I, + formatted_raw_ostream &OS) override { + const char *Name; + const Value *Value; + if (const CallInst *CI = dyn_cast<CallInst>(I)) { + Name = "call"; + Value = CI->getCalledOperand(); + } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) { + Name = "invoke"; + Value = II->getCalledOperand(); + } else { + // Could demangle more operations, e. g. + // `store %place, @function`. + return; + } + + if (!Value->hasName()) { + return; + } + + StringRef Demangled = CallDemangle(Value->getName()); + if (Demangled.empty()) { + return; + } + + OS << "; " << Name << " " << Demangled << "\n"; + } +}; + +} // namespace + +extern "C" LLVMRustResult +LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) { + std::string ErrorInfo; + std::error_code EC; + raw_fd_ostream OS(Path, EC, sys::fs::OF_None); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + + RustAssemblyAnnotationWriter AAW(Demangle); + formatted_raw_ostream FOS(OS); + unwrap(M)->print(FOS, &AAW); + + return LLVMRustResult::Success; +} + +extern "C" void LLVMRustPrintPasses() { + LLVMInitializePasses(); + struct MyListener : PassRegistrationListener { + void passEnumerate(const PassInfo *Info) { + StringRef PassArg = Info->getPassArgument(); + StringRef PassName = Info->getPassName(); + if (!PassArg.empty()) { + // These unsigned->signed casts could theoretically overflow, but + // realistically never will (and even if, the result is implementation + // defined rather plain UB). + printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(), + (int)PassName.size(), PassName.data()); + } + } + } Listener; + + PassRegistry *PR = PassRegistry::getPassRegistry(); + PR->enumerateWith(&Listener); +} + +extern "C" void LLVMRustAddAlwaysInlinePass(LLVMPassManagerBuilderRef PMBR, + bool AddLifetimes) { + unwrap(PMBR)->Inliner = llvm::createAlwaysInlinerLegacyPass(AddLifetimes); +} + +extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, + size_t Len) { + llvm::legacy::PassManager passes; + + auto PreserveFunctions = [=](const GlobalValue &GV) { + for (size_t I = 0; I < Len; I++) { + if (GV.getName() == Symbols[I]) { + return true; + } + } + return false; + }; + + passes.add(llvm::createInternalizePass(PreserveFunctions)); + + passes.run(*unwrap(M)); +} + +extern "C" void +LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, + LLVMTargetMachineRef TMR) { + TargetMachine *Target = unwrap(TMR); + unwrap(Module)->setDataLayout(Target->createDataLayout()); +} + +extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { + unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); +} + +extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { + unwrap(M)->setPIELevel(PIELevel::Level::Large); +} + +extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, + LLVMRustCodeModel Model) { + auto CM = fromRust(Model); + if (!CM.hasValue()) + return; + unwrap(M)->setCodeModel(*CM); +} + +// Here you'll find an implementation of ThinLTO as used by the Rust compiler +// right now. This ThinLTO support is only enabled on "recent ish" versions of +// LLVM, and otherwise it's just blanket rejected from other compilers. +// +// Most of this implementation is straight copied from LLVM. At the time of +// this writing it wasn't *quite* suitable to reuse more code from upstream +// for our purposes, but we should strive to upstream this support once it's +// ready to go! I figure we may want a bit of testing locally first before +// sending this upstream to LLVM. I hear though they're quite eager to receive +// feedback like this! +// +// If you're reading this code and wondering "what in the world" or you're +// working "good lord by LLVM upgrade is *still* failing due to these bindings" +// then fear not! (ok maybe fear a little). All code here is mostly based +// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. +// +// You'll find that the general layout here roughly corresponds to the `run` +// method in that file as well as `ProcessThinLTOModule`. Functions are +// specifically commented below as well, but if you're updating this code +// or otherwise trying to understand it, the LLVM source will be useful in +// interpreting the mysteries within. +// +// Otherwise I'll apologize in advance, it probably requires a relatively +// significant investment on your part to "truly understand" what's going on +// here. Not saying I do myself, but it took me awhile staring at LLVM's source +// and various online resources about ThinLTO to make heads or tails of all +// this. + +// This is a shared data structure which *must* be threadsafe to share +// read-only amongst threads. This also corresponds basically to the arguments +// of the `ProcessThinLTOModule` function in the LLVM source. +struct LLVMRustThinLTOData { + // The combined index that is the global analysis over all modules we're + // performing ThinLTO for. This is mostly managed by LLVM. + ModuleSummaryIndex Index; + + // All modules we may look at, stored as in-memory serialized versions. This + // is later used when inlining to ensure we can extract any module to inline + // from. + StringMap<MemoryBufferRef> ModuleMap; + + // A set that we manage of everything we *don't* want internalized. Note that + // this includes all transitive references right now as well, but it may not + // always! + DenseSet<GlobalValue::GUID> GUIDPreservedSymbols; + + // Not 100% sure what these are, but they impact what's internalized and + // what's inlined across modules, I believe. + StringMap<FunctionImporter::ImportMapTy> ImportLists; + StringMap<FunctionImporter::ExportSetTy> ExportLists; + StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; + StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; + + LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} +}; + +// Just an argument to the `LLVMRustCreateThinLTOData` function below. +struct LLVMRustThinLTOModule { + const char *identifier; + const char *data; + size_t len; +}; + +// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it +// does. +static const GlobalValueSummary * +getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { + auto StrongDefForLinker = llvm::find_if( + GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { + auto Linkage = Summary->linkage(); + return !GlobalValue::isAvailableExternallyLinkage(Linkage) && + !GlobalValue::isWeakForLinker(Linkage); + }); + if (StrongDefForLinker != GVSummaryList.end()) + return StrongDefForLinker->get(); + + auto FirstDefForLinker = llvm::find_if( + GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) { + auto Linkage = Summary->linkage(); + return !GlobalValue::isAvailableExternallyLinkage(Linkage); + }); + if (FirstDefForLinker == GVSummaryList.end()) + return nullptr; + return FirstDefForLinker->get(); +} + +// The main entry point for creating the global ThinLTO analysis. The structure +// here is basically the same as before threads are spawned in the `run` +// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. +extern "C" LLVMRustThinLTOData* +LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, + int num_modules, + const char **preserved_symbols, + int num_symbols) { + auto Ret = std::make_unique<LLVMRustThinLTOData>(); + + // Load each module's summary and merge it into one combined index + for (int i = 0; i < num_modules; i++) { + auto module = &modules[i]; + StringRef buffer(module->data, module->len); + MemoryBufferRef mem_buffer(buffer, module->identifier); + + Ret->ModuleMap[module->identifier] = mem_buffer; + + if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index, i)) { + LLVMRustSetLastError(toString(std::move(Err)).c_str()); + return nullptr; + } + } + + // Collect for each module the list of function it defines (GUID -> Summary) + Ret->Index.collectDefinedGVSummariesPerModule(Ret->ModuleToDefinedGVSummaries); + + // Convert the preserved symbols set from string to GUID, this is then needed + // for internalization. + for (int i = 0; i < num_symbols; i++) { + auto GUID = GlobalValue::getGUID(preserved_symbols[i]); + Ret->GUIDPreservedSymbols.insert(GUID); + } + + // Collect the import/export lists for all modules from the call-graph in the + // combined index + // + // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` + auto deadIsPrevailing = [&](GlobalValue::GUID G) { + return PrevailingType::Unknown; + }; + // We don't have a complete picture in our use of ThinLTO, just our immediate + // crate, so we need `ImportEnabled = false` to limit internalization. + // Otherwise, we sometimes lose `static` values -- see #60184. + computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, + deadIsPrevailing, /* ImportEnabled = */ false); + ComputeCrossModuleImport( + Ret->Index, + Ret->ModuleToDefinedGVSummaries, + Ret->ImportLists, + Ret->ExportLists + ); + + // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it + // impacts the caching. + // + // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this + // being lifted from `lib/LTO/LTO.cpp` as well + DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; + for (auto &I : Ret->Index) { + if (I.second.SummaryList.size() > 1) + PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second.SummaryList); + } + auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { + const auto &Prevailing = PrevailingCopy.find(GUID); + if (Prevailing == PrevailingCopy.end()) + return true; + return Prevailing->second == S; + }; + auto recordNewLinkage = [&](StringRef ModuleIdentifier, + GlobalValue::GUID GUID, + GlobalValue::LinkageTypes NewLinkage) { + Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; + }; + +#if LLVM_VERSION_GE(13,0) + // Uses FromPrevailing visibility scheme which works for many binary + // formats. We probably could and should use ELF visibility scheme for many of + // our targets, however. + lto::Config conf; + thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, recordNewLinkage, + Ret->GUIDPreservedSymbols); +#else + thinLTOResolvePrevailingInIndex(Ret->Index, isPrevailing, recordNewLinkage, + Ret->GUIDPreservedSymbols); +#endif + // Here we calculate an `ExportedGUIDs` set for use in the `isExported` + // callback below. This callback below will dictate the linkage for all + // summaries in the index, and we basically just only want to ensure that dead + // symbols are internalized. Otherwise everything that's already external + // linkage will stay as external, and internal will stay as internal. + std::set<GlobalValue::GUID> ExportedGUIDs; + for (auto &List : Ret->Index) { + for (auto &GVS: List.second.SummaryList) { + if (GlobalValue::isLocalLinkage(GVS->linkage())) + continue; + auto GUID = GVS->getOriginalName(); + if (GVS->flags().Live) + ExportedGUIDs.insert(GUID); + } + } + auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { + const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); + return (ExportList != Ret->ExportLists.end() && + ExportList->second.count(VI)) || + ExportedGUIDs.count(VI.getGUID()); + }; + thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); + + return Ret.release(); +} + +extern "C" void +LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { + delete Data; +} + +// Below are the various passes that happen *per module* when doing ThinLTO. +// +// In other words, these are the functions that are all run concurrently +// with one another, one per module. The passes here correspond to the analysis +// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the +// `ProcessThinLTOModule` function. Here they're split up into separate steps +// so rustc can save off the intermediate bytecode between each step. + +static bool +clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { + // When linking an ELF shared object, dso_local should be dropped. We + // conservatively do this for -fpic. + bool ClearDSOLocalOnDeclarations = + TM.getTargetTriple().isOSBinFormatELF() && + TM.getRelocationModel() != Reloc::Static && + Mod.getPIELevel() == PIELevel::Default; + return ClearDSOLocalOnDeclarations; +} + +extern "C" bool +LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M, + LLVMTargetMachineRef TM) { + Module &Mod = *unwrap(M); + TargetMachine &Target = *unwrap(TM); + + bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); + bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); + + if (error) { + LLVMRustSetLastError("renameModuleForThinLTO failed"); + return false; + } + return true; +} + +extern "C" bool +LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { + Module &Mod = *unwrap(M); + const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); +#if LLVM_VERSION_GE(14, 0) + thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); +#else + thinLTOResolvePrevailingInModule(Mod, DefinedGlobals); +#endif + return true; +} + +extern "C" bool +LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef M) { + Module &Mod = *unwrap(M); + const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); + thinLTOInternalizeModule(Mod, DefinedGlobals); + return true; +} + +extern "C" bool +LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M, + LLVMTargetMachineRef TM) { + Module &Mod = *unwrap(M); + TargetMachine &Target = *unwrap(TM); + + const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); + auto Loader = [&](StringRef Identifier) { + const auto &Memory = Data->ModuleMap.lookup(Identifier); + auto &Context = Mod.getContext(); + auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); + + if (!MOrErr) + return MOrErr; + + // The rest of this closure is a workaround for + // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports + // we accidentally import wasm custom sections into different modules, + // duplicating them by in the final output artifact. + // + // The issue is worked around here by manually removing the + // `wasm.custom_sections` named metadata node from any imported module. This + // we know isn't used by any optimization pass so there's no need for it to + // be imported. + // + // Note that the metadata is currently lazily loaded, so we materialize it + // here before looking up if there's metadata inside. The `FunctionImporter` + // will immediately materialize metadata anyway after an import, so this + // shouldn't be a perf hit. + if (Error Err = (*MOrErr)->materializeMetadata()) { + Expected<std::unique_ptr<Module>> Ret(std::move(Err)); + return Ret; + } + + auto *WasmCustomSections = (*MOrErr)->getNamedMetadata("wasm.custom_sections"); + if (WasmCustomSections) + WasmCustomSections->eraseFromParent(); + + return MOrErr; + }; + bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); + FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal); + Expected<bool> Result = Importer.importFunctions(Mod, ImportList); + if (!Result) { + LLVMRustSetLastError(toString(Result.takeError()).c_str()); + return false; + } + return true; +} + +// This struct and various functions are sort of a hack right now, but the +// problem is that we've got in-memory LLVM modules after we generate and +// optimize all codegen-units for one compilation in rustc. To be compatible +// with the LTO support above we need to serialize the modules plus their +// ThinLTO summary into memory. +// +// This structure is basically an owned version of a serialize module, with +// a ThinLTO summary attached. +struct LLVMRustThinLTOBuffer { + std::string data; +}; + +extern "C" LLVMRustThinLTOBuffer* +LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin) { + auto Ret = std::make_unique<LLVMRustThinLTOBuffer>(); + { + raw_string_ostream OS(Ret->data); + { + legacy::PassManager PM; + if (is_thin) { + PM.add(createWriteThinLTOBitcodePass(OS)); + } else { + PM.add(createBitcodeWriterPass(OS)); + } + PM.run(*unwrap(M)); + } + } + return Ret.release(); +} + +extern "C" void +LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { + delete Buffer; +} + +extern "C" const void* +LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->data.data(); +} + +extern "C" size_t +LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->data.length(); +} + +// This is what we used to parse upstream bitcode for actual ThinLTO +// processing. We'll call this once per module optimized through ThinLTO, and +// it'll be called concurrently on many threads. +extern "C" LLVMModuleRef +LLVMRustParseBitcodeForLTO(LLVMContextRef Context, + const char *data, + size_t len, + const char *identifier) { + StringRef Data(data, len); + MemoryBufferRef Buffer(Data, identifier); + unwrap(Context)->enableDebugTypeODRUniquing(); + Expected<std::unique_ptr<Module>> SrcOrError = + parseBitcodeFile(Buffer, *unwrap(Context)); + if (!SrcOrError) { + LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); + return nullptr; + } + return wrap(std::move(*SrcOrError).release()); +} + +// Find the bitcode section in the object file data and return it as a slice. +// Fail if the bitcode section is present but empty. +// +// On success, the return value is the pointer to the start of the slice and +// `out_len` is filled with the (non-zero) length. On failure, the return value +// is `nullptr` and `out_len` is set to zero. +extern "C" const char* +LLVMRustGetBitcodeSliceFromObjectData(const char *data, + size_t len, + size_t *out_len) { + *out_len = 0; + + StringRef Data(data, len); + MemoryBufferRef Buffer(Data, ""); // The id is unused. + + Expected<MemoryBufferRef> BitcodeOrError = + object::IRObjectFile::findBitcodeInMemBuffer(Buffer); + if (!BitcodeOrError) { + LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str()); + return nullptr; + } + + *out_len = BitcodeOrError->getBufferSize(); + return BitcodeOrError->getBufferStart(); +} + +// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See +// the comment in `back/lto.rs` for why this exists. +extern "C" void +LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, + DICompileUnit **A, + DICompileUnit **B) { + Module *M = unwrap(Mod); + DICompileUnit **Cur = A; + DICompileUnit **Next = B; + for (DICompileUnit *CU : M->debug_compile_units()) { + *Cur = CU; + Cur = Next; + Next = nullptr; + if (Cur == nullptr) + break; + } +} + +// Rewrite all `DICompileUnit` pointers to the `DICompileUnit` specified. See +// the comment in `back/lto.rs` for why this exists. +extern "C" void +LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) { + Module *M = unwrap(Mod); + + // If the original source module didn't have a `DICompileUnit` then try to + // merge all the existing compile units. If there aren't actually any though + // then there's not much for us to do so return. + if (Unit == nullptr) { + for (DICompileUnit *CU : M->debug_compile_units()) { + Unit = CU; + break; + } + if (Unit == nullptr) + return; + } + + // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and + // process it recursively. Note that we used to specifically iterate over + // instructions to ensure we feed everything into it, but `processModule` + // started doing this the same way in LLVM 7 (commit d769eb36ab2b8). + DebugInfoFinder Finder; + Finder.processModule(*M); + + // After we've found all our debuginfo, rewrite all subprograms to point to + // the same `DICompileUnit`. + for (auto &F : Finder.subprograms()) { + F->replaceUnit(Unit); + } + + // Erase any other references to other `DICompileUnit` instances, the verifier + // will later ensure that we don't actually have any other stale references to + // worry about. + auto *MD = M->getNamedMetadata("llvm.dbg.cu"); + MD->clearOperands(); + MD->addOperand(Unit); +} + +// Computes the LTO cache key for the provided 'ModId' in the given 'Data', +// storing the result in 'KeyOut'. +// Currently, this cache key is a SHA-1 hash of anything that could affect +// the result of optimizing this module (e.g. module imports, exports, liveness +// of access globals, etc). +// The precise details are determined by LLVM in `computeLTOCacheKey`, which is +// used during the normal linker-plugin incremental thin-LTO process. +extern "C" void +LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, const char *ModId, LLVMRustThinLTOData *Data) { + SmallString<40> Key; + llvm::lto::Config conf; + const auto &ImportList = Data->ImportLists.lookup(ModId); + const auto &ExportList = Data->ExportLists.lookup(ModId); + const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); + const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); + std::set<GlobalValue::GUID> CfiFunctionDefs; + std::set<GlobalValue::GUID> CfiFunctionDecls; + + // Based on the 'InProcessThinBackend' constructor in LLVM + for (auto &Name : Data->Index.cfiFunctionDefs()) + CfiFunctionDefs.insert( + GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); + for (auto &Name : Data->Index.cfiFunctionDecls()) + CfiFunctionDecls.insert( + GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); + + llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, + ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls + ); + + LLVMRustStringWriteImpl(KeyOut, Key.c_str(), Key.size()); +} diff --git a/compiler/rustc_llvm/llvm-wrapper/README b/compiler/rustc_llvm/llvm-wrapper/README new file mode 100644 index 000000000..e1c6dd07d --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/README @@ -0,0 +1,16 @@ +This directory currently contains some LLVM support code. This will generally +be sent upstream to LLVM in time; for now it lives here. + +NOTE: the LLVM C++ ABI is subject to between-version breakage and must *never* +be exposed to Rust. To allow for easy auditing of that, all Rust-exposed types +must be typedef-ed as "LLVMXyz", or "LLVMRustXyz" if they were defined here. + +Functions that return a failure status and leave the error in +the LLVM last error should return an LLVMRustResult rather than an +int or anything to avoid confusion. + +When translating enums, add a single `Other` variant as the first +one to allow for new variants to be added. It should abort when used +as an input. + +All other types must not be typedef-ed as such. diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp new file mode 100644 index 000000000..5f5b5de79 --- /dev/null +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -0,0 +1,1972 @@ +#include "LLVMWrapper.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DiagnosticHandler.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsARM.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Object/Archive.h" +#include "llvm/Object/COFFImportFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Pass.h" +#include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm/Support/Signals.h" +#include "llvm/ADT/Optional.h" + +#include <iostream> + +//===----------------------------------------------------------------------=== +// +// This file defines alternate interfaces to core functions that are more +// readily callable by Rust's FFI. +// +//===----------------------------------------------------------------------=== + +using namespace llvm; +using namespace llvm::sys; +using namespace llvm::object; + +// LLVMAtomicOrdering is already an enum - don't create another +// one. +static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering) { + switch (Ordering) { + case LLVMAtomicOrderingNotAtomic: + return AtomicOrdering::NotAtomic; + case LLVMAtomicOrderingUnordered: + return AtomicOrdering::Unordered; + case LLVMAtomicOrderingMonotonic: + return AtomicOrdering::Monotonic; + case LLVMAtomicOrderingAcquire: + return AtomicOrdering::Acquire; + case LLVMAtomicOrderingRelease: + return AtomicOrdering::Release; + case LLVMAtomicOrderingAcquireRelease: + return AtomicOrdering::AcquireRelease; + case LLVMAtomicOrderingSequentiallyConsistent: + return AtomicOrdering::SequentiallyConsistent; + } + + report_fatal_error("Invalid LLVMAtomicOrdering value!"); +} + +static LLVM_THREAD_LOCAL char *LastError; + +// Custom error handler for fatal LLVM errors. +// +// Notably it exits the process with code 101, unlike LLVM's default of 1. +static void FatalErrorHandler(void *UserData, +#if LLVM_VERSION_LT(14, 0) + const std::string& Reason, +#else + const char* Reason, +#endif + bool GenCrashDiag) { + // Do the same thing that the default error handler does. + std::cerr << "LLVM ERROR: " << Reason << std::endl; + + // Since this error handler exits the process, we have to run any cleanup that + // LLVM would run after handling the error. This might change with an LLVM + // upgrade. + sys::RunInterruptHandlers(); + + exit(101); +} + +extern "C" void LLVMRustInstallFatalErrorHandler() { + install_fatal_error_handler(FatalErrorHandler); +} + +extern "C" void LLVMRustDisableSystemDialogsOnCrash() { + sys::DisableSystemDialogsOnCrash(); +} + +extern "C" char *LLVMRustGetLastError(void) { + char *Ret = LastError; + LastError = nullptr; + return Ret; +} + +extern "C" void LLVMRustSetLastError(const char *Err) { + free((void *)LastError); + LastError = strdup(Err); +} + +extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { + auto ctx = new LLVMContext(); + ctx->setDiscardValueNames(shouldDiscardNames); + return wrap(ctx); +} + +extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, + const char *Triple) { + unwrap(M)->setTargetTriple(Triple::normalize(Triple)); +} + +extern "C" void LLVMRustPrintPassTimings() { + raw_fd_ostream OS(2, false); // stderr. + TimerGroup::printAll(OS); +} + +extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, + size_t NameLen) { + return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen))); +} + +extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M, + const char *Name, + size_t NameLen, + LLVMTypeRef FunctionTy) { + return wrap(unwrap(M) + ->getOrInsertFunction(StringRef(Name, NameLen), + unwrap<FunctionType>(FunctionTy)) + .getCallee() + ); +} + +extern "C" LLVMValueRef +LLVMRustGetOrInsertGlobal(LLVMModuleRef M, const char *Name, size_t NameLen, LLVMTypeRef Ty) { + Module *Mod = unwrap(M); + StringRef NameRef(Name, NameLen); + + // We don't use Module::getOrInsertGlobal because that returns a Constant*, + // which may either be the real GlobalVariable*, or a constant bitcast of it + // if our type doesn't match the original declaration. We always want the + // GlobalVariable* so we can access linkage, visibility, etc. + GlobalVariable *GV = Mod->getGlobalVariable(NameRef, true); + if (!GV) + GV = new GlobalVariable(*Mod, unwrap(Ty), false, + GlobalValue::ExternalLinkage, nullptr, NameRef); + return wrap(GV); +} + +extern "C" LLVMValueRef +LLVMRustInsertPrivateGlobal(LLVMModuleRef M, LLVMTypeRef Ty) { + return wrap(new GlobalVariable(*unwrap(M), + unwrap(Ty), + false, + GlobalValue::PrivateLinkage, + nullptr)); +} + +extern "C" LLVMTypeRef LLVMRustMetadataTypeInContext(LLVMContextRef C) { + return wrap(Type::getMetadataTy(*unwrap(C))); +} + +static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) { + switch (Kind) { + case AlwaysInline: + return Attribute::AlwaysInline; + case ByVal: + return Attribute::ByVal; + case Cold: + return Attribute::Cold; + case InlineHint: + return Attribute::InlineHint; + case MinSize: + return Attribute::MinSize; + case Naked: + return Attribute::Naked; + case NoAlias: + return Attribute::NoAlias; + case NoCapture: + return Attribute::NoCapture; + case NoCfCheck: + return Attribute::NoCfCheck; + case NoInline: + return Attribute::NoInline; + case NonNull: + return Attribute::NonNull; + case NoRedZone: + return Attribute::NoRedZone; + case NoReturn: + return Attribute::NoReturn; + case NoUnwind: + return Attribute::NoUnwind; + case OptimizeForSize: + return Attribute::OptimizeForSize; + case ReadOnly: + return Attribute::ReadOnly; + case SExt: + return Attribute::SExt; + case StructRet: + return Attribute::StructRet; + case UWTable: + return Attribute::UWTable; + case ZExt: + return Attribute::ZExt; + case InReg: + return Attribute::InReg; + case SanitizeThread: + return Attribute::SanitizeThread; + case SanitizeAddress: + return Attribute::SanitizeAddress; + case SanitizeMemory: + return Attribute::SanitizeMemory; + case NonLazyBind: + return Attribute::NonLazyBind; + case OptimizeNone: + return Attribute::OptimizeNone; + case ReturnsTwice: + return Attribute::ReturnsTwice; + case ReadNone: + return Attribute::ReadNone; + case InaccessibleMemOnly: + return Attribute::InaccessibleMemOnly; + case SanitizeHWAddress: + return Attribute::SanitizeHWAddress; + case WillReturn: + return Attribute::WillReturn; + case StackProtectReq: + return Attribute::StackProtectReq; + case StackProtectStrong: + return Attribute::StackProtectStrong; + case StackProtect: + return Attribute::StackProtect; + case NoUndef: + return Attribute::NoUndef; + case SanitizeMemTag: + return Attribute::SanitizeMemTag; + case ShadowCallStack: + return Attribute::ShadowCallStack; + case AllocSize: + return Attribute::AllocSize; +#if LLVM_VERSION_GE(15, 0) + case AllocatedPointer: + return Attribute::AllocatedPointer; + case AllocAlign: + return Attribute::AllocAlign; +#endif + } + report_fatal_error("bad AttributeKind"); +} + +template<typename T> static inline void AddAttributes(T *t, unsigned Index, + LLVMAttributeRef *Attrs, size_t AttrsLen) { + AttributeList PAL = t->getAttributes(); + AttributeList PALNew; +#if LLVM_VERSION_LT(14, 0) + AttrBuilder B; + for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) + B.addAttribute(unwrap(Attr)); + PALNew = PAL.addAttributes(t->getContext(), Index, B); +#else + AttrBuilder B(t->getContext()); + for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen)) + B.addAttribute(unwrap(Attr)); + PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B); +#endif + t->setAttributes(PALNew); +} + +extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index, + LLVMAttributeRef *Attrs, size_t AttrsLen) { + Function *F = unwrap<Function>(Fn); + AddAttributes(F, Index, Attrs, AttrsLen); +} + +extern "C" void LLVMRustAddCallSiteAttributes(LLVMValueRef Instr, unsigned Index, + LLVMAttributeRef *Attrs, size_t AttrsLen) { + CallBase *Call = unwrap<CallBase>(Instr); + AddAttributes(Call, Index, Attrs, AttrsLen); +} + +extern "C" LLVMAttributeRef LLVMRustCreateAttrNoValue(LLVMContextRef C, + LLVMRustAttribute RustAttr) { + return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateAlignmentAttr(LLVMContextRef C, + uint64_t Bytes) { + return wrap(Attribute::getWithAlignment(*unwrap(C), llvm::Align(Bytes))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableAttr(LLVMContextRef C, + uint64_t Bytes) { + return wrap(Attribute::getWithDereferenceableBytes(*unwrap(C), Bytes)); +} + +extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableOrNullAttr(LLVMContextRef C, + uint64_t Bytes) { + return wrap(Attribute::getWithDereferenceableOrNullBytes(*unwrap(C), Bytes)); +} + +extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C, LLVMTypeRef Ty) { + return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, LLVMTypeRef Ty) { + return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateElementTypeAttr(LLVMContextRef C, LLVMTypeRef Ty) { +#if LLVM_VERSION_GE(15, 0) + return wrap(Attribute::get(*unwrap(C), Attribute::ElementType, unwrap(Ty))); +#else + report_fatal_error("Should not be needed on LLVM < 15"); +#endif +} + +extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Async) { +#if LLVM_VERSION_LT(15, 0) + return wrap(Attribute::get(*unwrap(C), Attribute::UWTable)); +#else + return wrap(Attribute::getWithUWTableKind( + *unwrap(C), Async ? UWTableKind::Async : UWTableKind::Sync)); +#endif +} + +extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) { + return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, None)); +} + +#if LLVM_VERSION_GE(15, 0) + +// These values **must** match ffi::AllocKindFlags. +// It _happens_ to match the LLVM values of llvm::AllocFnKind, +// but that's happenstance and we do explicit conversions before +// passing them to LLVM. +enum class LLVMRustAllocKindFlags : uint64_t { + Unknown = 0, + Alloc = 1, + Realloc = 1 << 1, + Free = 1 << 2, + Uninitialized = 1 << 3, + Zeroed = 1 << 4, + Aligned = 1 << 5, +}; + +static LLVMRustAllocKindFlags operator&(LLVMRustAllocKindFlags A, LLVMRustAllocKindFlags B) { + return static_cast<LLVMRustAllocKindFlags>(static_cast<uint64_t>(A) & + static_cast<uint64_t>(B)); +} + +static bool isSet(LLVMRustAllocKindFlags F) { return F != LLVMRustAllocKindFlags::Unknown; } + +static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) { + llvm::AllocFnKind AFK = llvm::AllocFnKind::Unknown; + if (isSet(F & LLVMRustAllocKindFlags::Alloc)) { + AFK |= llvm::AllocFnKind::Alloc; + } + if (isSet(F & LLVMRustAllocKindFlags::Realloc)) { + AFK |= llvm::AllocFnKind::Realloc; + } + if (isSet(F & LLVMRustAllocKindFlags::Free)) { + AFK |= llvm::AllocFnKind::Free; + } + if (isSet(F & LLVMRustAllocKindFlags::Uninitialized)) { + AFK |= llvm::AllocFnKind::Uninitialized; + } + if (isSet(F & LLVMRustAllocKindFlags::Zeroed)) { + AFK |= llvm::AllocFnKind::Zeroed; + } + if (isSet(F & LLVMRustAllocKindFlags::Aligned)) { + AFK |= llvm::AllocFnKind::Aligned; + } + return AFK; +} +#endif + +extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, uint64_t AllocKindArg) { +#if LLVM_VERSION_GE(15, 0) + return wrap(Attribute::get(*unwrap(C), Attribute::AllocKind, + static_cast<uint64_t>(allocKindFromRust(static_cast<LLVMRustAllocKindFlags>(AllocKindArg))))); +#else + report_fatal_error( + "allockind attributes are new in LLVM 15 and should not be used on older LLVMs"); +#endif +} + +// Enable a fast-math flag +// +// https://llvm.org/docs/LangRef.html#fast-math-flags +extern "C" void LLVMRustSetFastMath(LLVMValueRef V) { + if (auto I = dyn_cast<Instruction>(unwrap<Value>(V))) { + I->setFast(true); + } +} + +extern "C" LLVMValueRef +LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, + const char *Name, LLVMAtomicOrdering Order) { + Value *Ptr = unwrap(Source); + LoadInst *LI = unwrap(B)->CreateLoad(unwrap(Ty), Ptr, Name); + LI->setAtomic(fromRust(Order)); + return wrap(LI); +} + +extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, + LLVMValueRef V, + LLVMValueRef Target, + LLVMAtomicOrdering Order) { + StoreInst *SI = unwrap(B)->CreateStore(unwrap(V), unwrap(Target)); + SI->setAtomic(fromRust(Order)); + return wrap(SI); +} + +// FIXME: Use the C-API LLVMBuildAtomicCmpXchg and LLVMSetWeak +// once we raise our minimum support to LLVM 10. +extern "C" LLVMValueRef +LLVMRustBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Target, + LLVMValueRef Old, LLVMValueRef Source, + LLVMAtomicOrdering Order, + LLVMAtomicOrdering FailureOrder, LLVMBool Weak) { +#if LLVM_VERSION_GE(13,0) + // Rust probably knows the alignment of the target value and should be able to + // specify something more precise than MaybeAlign here. See also + // https://reviews.llvm.org/D97224 which may be a useful reference. + AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( + unwrap(Target), unwrap(Old), unwrap(Source), llvm::MaybeAlign(), fromRust(Order), + fromRust(FailureOrder)); +#else + AtomicCmpXchgInst *ACXI = unwrap(B)->CreateAtomicCmpXchg( + unwrap(Target), unwrap(Old), unwrap(Source), fromRust(Order), + fromRust(FailureOrder)); +#endif + ACXI->setWeak(Weak); + return wrap(ACXI); +} + +enum class LLVMRustSynchronizationScope { + SingleThread, + CrossThread, +}; + +static SyncScope::ID fromRust(LLVMRustSynchronizationScope Scope) { + switch (Scope) { + case LLVMRustSynchronizationScope::SingleThread: + return SyncScope::SingleThread; + case LLVMRustSynchronizationScope::CrossThread: + return SyncScope::System; + default: + report_fatal_error("bad SynchronizationScope."); + } +} + +extern "C" LLVMValueRef +LLVMRustBuildAtomicFence(LLVMBuilderRef B, LLVMAtomicOrdering Order, + LLVMRustSynchronizationScope Scope) { + return wrap(unwrap(B)->CreateFence(fromRust(Order), fromRust(Scope))); +} + +enum class LLVMRustAsmDialect { + Att, + Intel, +}; + +static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) { + switch (Dialect) { + case LLVMRustAsmDialect::Att: + return InlineAsm::AD_ATT; + case LLVMRustAsmDialect::Intel: + return InlineAsm::AD_Intel; + default: + report_fatal_error("bad AsmDialect."); + } +} + +extern "C" LLVMValueRef +LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, + char *Constraints, size_t ConstraintsLen, + LLVMBool HasSideEffects, LLVMBool IsAlignStack, + LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { +#if LLVM_VERSION_GE(13, 0) + return wrap(InlineAsm::get(unwrap<FunctionType>(Ty), + StringRef(AsmString, AsmStringLen), + StringRef(Constraints, ConstraintsLen), + HasSideEffects, IsAlignStack, + fromRust(Dialect), CanThrow)); +#else + return wrap(InlineAsm::get(unwrap<FunctionType>(Ty), + StringRef(AsmString, AsmStringLen), + StringRef(Constraints, ConstraintsLen), + HasSideEffects, IsAlignStack, + fromRust(Dialect))); +#endif +} + +extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, + size_t ConstraintsLen) { +#if LLVM_VERSION_LT(15, 0) + return InlineAsm::Verify(unwrap<FunctionType>(Ty), + StringRef(Constraints, ConstraintsLen)); +#else + // llvm::Error converts to true if it is an error. + return !llvm::errorToBool(InlineAsm::verify( + unwrap<FunctionType>(Ty), StringRef(Constraints, ConstraintsLen))); +#endif +} + +extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm, + size_t AsmLen) { + unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen)); +} + +typedef DIBuilder *LLVMRustDIBuilderRef; + +template <typename DIT> DIT *unwrapDIPtr(LLVMMetadataRef Ref) { + return (DIT *)(Ref ? unwrap<MDNode>(Ref) : nullptr); +} + +#define DIDescriptor DIScope +#define DIArray DINodeArray +#define unwrapDI unwrapDIPtr + +// These values **must** match debuginfo::DIFlags! They also *happen* +// to match LLVM, but that isn't required as we do giant sets of +// matching below. The value shouldn't be directly passed to LLVM. +enum class LLVMRustDIFlags : uint32_t { + FlagZero = 0, + FlagPrivate = 1, + FlagProtected = 2, + FlagPublic = 3, + FlagFwdDecl = (1 << 2), + FlagAppleBlock = (1 << 3), + FlagBlockByrefStruct = (1 << 4), + FlagVirtual = (1 << 5), + FlagArtificial = (1 << 6), + FlagExplicit = (1 << 7), + FlagPrototyped = (1 << 8), + FlagObjcClassComplete = (1 << 9), + FlagObjectPointer = (1 << 10), + FlagVector = (1 << 11), + FlagStaticMember = (1 << 12), + FlagLValueReference = (1 << 13), + FlagRValueReference = (1 << 14), + FlagExternalTypeRef = (1 << 15), + FlagIntroducedVirtual = (1 << 18), + FlagBitField = (1 << 19), + FlagNoReturn = (1 << 20), + // Do not add values that are not supported by the minimum LLVM + // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def +}; + +inline LLVMRustDIFlags operator&(LLVMRustDIFlags A, LLVMRustDIFlags B) { + return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(A) & + static_cast<uint32_t>(B)); +} + +inline LLVMRustDIFlags operator|(LLVMRustDIFlags A, LLVMRustDIFlags B) { + return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(A) | + static_cast<uint32_t>(B)); +} + +inline LLVMRustDIFlags &operator|=(LLVMRustDIFlags &A, LLVMRustDIFlags B) { + return A = A | B; +} + +inline bool isSet(LLVMRustDIFlags F) { return F != LLVMRustDIFlags::FlagZero; } + +inline LLVMRustDIFlags visibility(LLVMRustDIFlags F) { + return static_cast<LLVMRustDIFlags>(static_cast<uint32_t>(F) & 0x3); +} + +static DINode::DIFlags fromRust(LLVMRustDIFlags Flags) { + DINode::DIFlags Result = DINode::DIFlags::FlagZero; + + switch (visibility(Flags)) { + case LLVMRustDIFlags::FlagPrivate: + Result |= DINode::DIFlags::FlagPrivate; + break; + case LLVMRustDIFlags::FlagProtected: + Result |= DINode::DIFlags::FlagProtected; + break; + case LLVMRustDIFlags::FlagPublic: + Result |= DINode::DIFlags::FlagPublic; + break; + default: + // The rest are handled below + break; + } + + if (isSet(Flags & LLVMRustDIFlags::FlagFwdDecl)) { + Result |= DINode::DIFlags::FlagFwdDecl; + } + if (isSet(Flags & LLVMRustDIFlags::FlagAppleBlock)) { + Result |= DINode::DIFlags::FlagAppleBlock; + } + if (isSet(Flags & LLVMRustDIFlags::FlagVirtual)) { + Result |= DINode::DIFlags::FlagVirtual; + } + if (isSet(Flags & LLVMRustDIFlags::FlagArtificial)) { + Result |= DINode::DIFlags::FlagArtificial; + } + if (isSet(Flags & LLVMRustDIFlags::FlagExplicit)) { + Result |= DINode::DIFlags::FlagExplicit; + } + if (isSet(Flags & LLVMRustDIFlags::FlagPrototyped)) { + Result |= DINode::DIFlags::FlagPrototyped; + } + if (isSet(Flags & LLVMRustDIFlags::FlagObjcClassComplete)) { + Result |= DINode::DIFlags::FlagObjcClassComplete; + } + if (isSet(Flags & LLVMRustDIFlags::FlagObjectPointer)) { + Result |= DINode::DIFlags::FlagObjectPointer; + } + if (isSet(Flags & LLVMRustDIFlags::FlagVector)) { + Result |= DINode::DIFlags::FlagVector; + } + if (isSet(Flags & LLVMRustDIFlags::FlagStaticMember)) { + Result |= DINode::DIFlags::FlagStaticMember; + } + if (isSet(Flags & LLVMRustDIFlags::FlagLValueReference)) { + Result |= DINode::DIFlags::FlagLValueReference; + } + if (isSet(Flags & LLVMRustDIFlags::FlagRValueReference)) { + Result |= DINode::DIFlags::FlagRValueReference; + } + if (isSet(Flags & LLVMRustDIFlags::FlagIntroducedVirtual)) { + Result |= DINode::DIFlags::FlagIntroducedVirtual; + } + if (isSet(Flags & LLVMRustDIFlags::FlagBitField)) { + Result |= DINode::DIFlags::FlagBitField; + } + if (isSet(Flags & LLVMRustDIFlags::FlagNoReturn)) { + Result |= DINode::DIFlags::FlagNoReturn; + } + + return Result; +} + +// These values **must** match debuginfo::DISPFlags! They also *happen* +// to match LLVM, but that isn't required as we do giant sets of +// matching below. The value shouldn't be directly passed to LLVM. +enum class LLVMRustDISPFlags : uint32_t { + SPFlagZero = 0, + SPFlagVirtual = 1, + SPFlagPureVirtual = 2, + SPFlagLocalToUnit = (1 << 2), + SPFlagDefinition = (1 << 3), + SPFlagOptimized = (1 << 4), + SPFlagMainSubprogram = (1 << 5), + // Do not add values that are not supported by the minimum LLVM + // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def + // (In LLVM < 8, createFunction supported these as separate bool arguments.) +}; + +inline LLVMRustDISPFlags operator&(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { + return static_cast<LLVMRustDISPFlags>(static_cast<uint32_t>(A) & + static_cast<uint32_t>(B)); +} + +inline LLVMRustDISPFlags operator|(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { + return static_cast<LLVMRustDISPFlags>(static_cast<uint32_t>(A) | + static_cast<uint32_t>(B)); +} + +inline LLVMRustDISPFlags &operator|=(LLVMRustDISPFlags &A, LLVMRustDISPFlags B) { + return A = A | B; +} + +inline bool isSet(LLVMRustDISPFlags F) { return F != LLVMRustDISPFlags::SPFlagZero; } + +inline LLVMRustDISPFlags virtuality(LLVMRustDISPFlags F) { + return static_cast<LLVMRustDISPFlags>(static_cast<uint32_t>(F) & 0x3); +} + +static DISubprogram::DISPFlags fromRust(LLVMRustDISPFlags SPFlags) { + DISubprogram::DISPFlags Result = DISubprogram::DISPFlags::SPFlagZero; + + switch (virtuality(SPFlags)) { + case LLVMRustDISPFlags::SPFlagVirtual: + Result |= DISubprogram::DISPFlags::SPFlagVirtual; + break; + case LLVMRustDISPFlags::SPFlagPureVirtual: + Result |= DISubprogram::DISPFlags::SPFlagPureVirtual; + break; + default: + // The rest are handled below + break; + } + + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagLocalToUnit)) { + Result |= DISubprogram::DISPFlags::SPFlagLocalToUnit; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagDefinition)) { + Result |= DISubprogram::DISPFlags::SPFlagDefinition; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagOptimized)) { + Result |= DISubprogram::DISPFlags::SPFlagOptimized; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagMainSubprogram)) { + Result |= DISubprogram::DISPFlags::SPFlagMainSubprogram; + } + + return Result; +} + +enum class LLVMRustDebugEmissionKind { + NoDebug, + FullDebug, + LineTablesOnly, +}; + +static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind) { + switch (Kind) { + case LLVMRustDebugEmissionKind::NoDebug: + return DICompileUnit::DebugEmissionKind::NoDebug; + case LLVMRustDebugEmissionKind::FullDebug: + return DICompileUnit::DebugEmissionKind::FullDebug; + case LLVMRustDebugEmissionKind::LineTablesOnly: + return DICompileUnit::DebugEmissionKind::LineTablesOnly; + default: + report_fatal_error("bad DebugEmissionKind."); + } +} + +enum class LLVMRustChecksumKind { + None, + MD5, + SHA1, + SHA256, +}; + +static Optional<DIFile::ChecksumKind> fromRust(LLVMRustChecksumKind Kind) { + switch (Kind) { + case LLVMRustChecksumKind::None: + return None; + case LLVMRustChecksumKind::MD5: + return DIFile::ChecksumKind::CSK_MD5; + case LLVMRustChecksumKind::SHA1: + return DIFile::ChecksumKind::CSK_SHA1; + case LLVMRustChecksumKind::SHA256: + return DIFile::ChecksumKind::CSK_SHA256; + default: + report_fatal_error("bad ChecksumKind."); + } +} + +extern "C" uint32_t LLVMRustDebugMetadataVersion() { + return DEBUG_METADATA_VERSION; +} + +extern "C" uint32_t LLVMRustVersionPatch() { return LLVM_VERSION_PATCH; } + +extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } + +extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } + +extern "C" void LLVMRustAddModuleFlag( + LLVMModuleRef M, + Module::ModFlagBehavior MergeBehavior, + const char *Name, + uint32_t Value) { + unwrap(M)->addModuleFlag(MergeBehavior, Name, Value); +} + +extern "C" bool LLVMRustHasModuleFlag(LLVMModuleRef M, const char *Name, + size_t Len) { + return unwrap(M)->getModuleFlag(StringRef(Name, Len)) != nullptr; +} + +extern "C" LLVMValueRef LLVMRustMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD) { + return wrap(MetadataAsValue::get(*unwrap(C), unwrap(MD))); +} + +extern "C" void LLVMRustGlobalAddMetadata( + LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { + unwrap<GlobalObject>(Global)->addMetadata(Kind, *unwrap<MDNode>(MD)); +} + +extern "C" LLVMRustDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { + return new DIBuilder(*unwrap(M)); +} + +extern "C" void LLVMRustDIBuilderDispose(LLVMRustDIBuilderRef Builder) { + delete Builder; +} + +extern "C" void LLVMRustDIBuilderFinalize(LLVMRustDIBuilderRef Builder) { + Builder->finalize(); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit( + LLVMRustDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef, + const char *Producer, size_t ProducerLen, bool isOptimized, + const char *Flags, unsigned RuntimeVer, + const char *SplitName, size_t SplitNameLen, + LLVMRustDebugEmissionKind Kind, + uint64_t DWOId, bool SplitDebugInlining) { + auto *File = unwrapDI<DIFile>(FileRef); + + return wrap(Builder->createCompileUnit(Lang, File, StringRef(Producer, ProducerLen), + isOptimized, Flags, RuntimeVer, + StringRef(SplitName, SplitNameLen), + fromRust(Kind), DWOId, SplitDebugInlining)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile( + LLVMRustDIBuilderRef Builder, + const char *Filename, size_t FilenameLen, + const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind, + const char *Checksum, size_t ChecksumLen) { + Optional<DIFile::ChecksumKind> llvmCSKind = fromRust(CSKind); + Optional<DIFile::ChecksumInfo<StringRef>> CSInfo{}; + if (llvmCSKind) + CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen}); + return wrap(Builder->createFile(StringRef(Filename, FilenameLen), + StringRef(Directory, DirectoryLen), + CSInfo)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateSubroutineType(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef ParameterTypes) { + return wrap(Builder->createSubroutineType( + DITypeRefArray(unwrap<MDTuple>(ParameterTypes)))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + const char *LinkageName, size_t LinkageNameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Ty, unsigned ScopeLine, LLVMRustDIFlags Flags, + LLVMRustDISPFlags SPFlags, LLVMValueRef MaybeFn, LLVMMetadataRef TParam, + LLVMMetadataRef Decl) { + DITemplateParameterArray TParams = + DITemplateParameterArray(unwrap<MDTuple>(TParam)); + DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); + DINode::DIFlags llvmFlags = fromRust(Flags); + DISubprogram *Sub = Builder->createFunction( + unwrapDI<DIScope>(Scope), + StringRef(Name, NameLen), + StringRef(LinkageName, LinkageNameLen), + unwrapDI<DIFile>(File), LineNo, + unwrapDI<DISubroutineType>(Ty), ScopeLine, llvmFlags, + llvmSPFlags, TParams, unwrapDIPtr<DISubprogram>(Decl)); + if (MaybeFn) + unwrap<Function>(MaybeFn)->setSubprogram(Sub); + return wrap(Sub); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateBasicType( + LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, + uint64_t SizeInBits, unsigned Encoding) { + return wrap(Builder->createBasicType(StringRef(Name, NameLen), SizeInBits, Encoding)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTypedef( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Type, const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Scope) { + return wrap(Builder->createTypedef( + unwrap<DIType>(Type), StringRef(Name, NameLen), unwrap<DIFile>(File), + LineNo, unwrapDIPtr<DIScope>(Scope))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef PointeeTy, + uint64_t SizeInBits, uint32_t AlignInBits, unsigned AddressSpace, + const char *Name, size_t NameLen) { + return wrap(Builder->createPointerType(unwrapDI<DIType>(PointeeTy), + SizeInBits, AlignInBits, + AddressSpace, + StringRef(Name, NameLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, + uint32_t AlignInBits, LLVMRustDIFlags Flags, + LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements, + unsigned RunTimeLang, LLVMMetadataRef VTableHolder, + const char *UniqueId, size_t UniqueIdLen) { + return wrap(Builder->createStructType( + unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), + unwrapDI<DIFile>(File), LineNumber, + SizeInBits, AlignInBits, fromRust(Flags), unwrapDI<DIType>(DerivedFrom), + DINodeArray(unwrapDI<MDTuple>(Elements)), RunTimeLang, + unwrapDI<DIType>(VTableHolder), StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, + uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Discriminator, + LLVMMetadataRef Elements, const char *UniqueId, size_t UniqueIdLen) { + return wrap(Builder->createVariantPart( + unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), + unwrapDI<DIFile>(File), LineNumber, + SizeInBits, AlignInBits, fromRust(Flags), unwrapDI<DIDerivedType>(Discriminator), + DINodeArray(unwrapDI<MDTuple>(Elements)), StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, + uint32_t AlignInBits, uint64_t OffsetInBits, LLVMRustDIFlags Flags, + LLVMMetadataRef Ty) { + return wrap(Builder->createMemberType(unwrapDI<DIDescriptor>(Scope), + StringRef(Name, NameLen), + unwrapDI<DIFile>(File), LineNo, + SizeInBits, AlignInBits, OffsetInBits, + fromRust(Flags), unwrapDI<DIType>(Ty))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, + uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant, + LLVMRustDIFlags Flags, LLVMMetadataRef Ty) { + llvm::ConstantInt* D = nullptr; + if (Discriminant) { + D = unwrap<llvm::ConstantInt>(Discriminant); + } + return wrap(Builder->createVariantMemberType(unwrapDI<DIDescriptor>(Scope), + StringRef(Name, NameLen), + unwrapDI<DIFile>(File), LineNo, + SizeInBits, AlignInBits, OffsetInBits, D, + fromRust(Flags), unwrapDI<DIType>(Ty))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlock( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + LLVMMetadataRef File, unsigned Line, unsigned Col) { + return wrap(Builder->createLexicalBlock(unwrapDI<DIDescriptor>(Scope), + unwrapDI<DIFile>(File), Line, Col)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateLexicalBlockFile(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef Scope, + LLVMMetadataRef File) { + return wrap(Builder->createLexicalBlockFile(unwrapDI<DIDescriptor>(Scope), + unwrapDI<DIFile>(File))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Context, + const char *Name, size_t NameLen, + const char *LinkageName, size_t LinkageNameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Ty, bool IsLocalToUnit, LLVMValueRef V, + LLVMMetadataRef Decl = nullptr, uint32_t AlignInBits = 0) { + llvm::GlobalVariable *InitVal = cast<llvm::GlobalVariable>(unwrap(V)); + + llvm::DIExpression *InitExpr = nullptr; + if (llvm::ConstantInt *IntVal = llvm::dyn_cast<llvm::ConstantInt>(InitVal)) { + InitExpr = Builder->createConstantValueExpression( + IntVal->getValue().getSExtValue()); + } else if (llvm::ConstantFP *FPVal = + llvm::dyn_cast<llvm::ConstantFP>(InitVal)) { + InitExpr = Builder->createConstantValueExpression( + FPVal->getValueAPF().bitcastToAPInt().getZExtValue()); + } + + llvm::DIGlobalVariableExpression *VarExpr = Builder->createGlobalVariableExpression( + unwrapDI<DIDescriptor>(Context), StringRef(Name, NameLen), + StringRef(LinkageName, LinkageNameLen), + unwrapDI<DIFile>(File), LineNo, unwrapDI<DIType>(Ty), IsLocalToUnit, + /* isDefined */ true, + InitExpr, unwrapDIPtr<MDNode>(Decl), + /* templateParams */ nullptr, + AlignInBits); + + InitVal->setMetadata("dbg", VarExpr); + + return wrap(VarExpr); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable( + LLVMRustDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMRustDIFlags Flags, + unsigned ArgNo, uint32_t AlignInBits) { + if (Tag == 0x100) { // DW_TAG_auto_variable + return wrap(Builder->createAutoVariable( + unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), + unwrapDI<DIFile>(File), LineNo, + unwrapDI<DIType>(Ty), AlwaysPreserve, fromRust(Flags), AlignInBits)); + } else { + return wrap(Builder->createParameterVariable( + unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), ArgNo, + unwrapDI<DIFile>(File), LineNo, + unwrapDI<DIType>(Ty), AlwaysPreserve, fromRust(Flags))); + } +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateArrayType(LLVMRustDIBuilderRef Builder, uint64_t Size, + uint32_t AlignInBits, LLVMMetadataRef Ty, + LLVMMetadataRef Subscripts) { + return wrap( + Builder->createArrayType(Size, AlignInBits, unwrapDI<DIType>(Ty), + DINodeArray(unwrapDI<MDTuple>(Subscripts)))); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderGetOrCreateSubrange(LLVMRustDIBuilderRef Builder, int64_t Lo, + int64_t Count) { + return wrap(Builder->getOrCreateSubrange(Lo, Count)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderGetOrCreateArray(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef *Ptr, unsigned Count) { + Metadata **DataValue = unwrap(Ptr); + return wrap( + Builder->getOrCreateArray(ArrayRef<Metadata *>(DataValue, Count)).get()); +} + +extern "C" LLVMValueRef LLVMRustDIBuilderInsertDeclareAtEnd( + LLVMRustDIBuilderRef Builder, LLVMValueRef V, LLVMMetadataRef VarInfo, + uint64_t *AddrOps, unsigned AddrOpsCount, LLVMMetadataRef DL, + LLVMBasicBlockRef InsertAtEnd) { + return wrap(Builder->insertDeclare( + unwrap(V), unwrap<DILocalVariable>(VarInfo), + Builder->createExpression(llvm::ArrayRef<uint64_t>(AddrOps, AddrOpsCount)), + DebugLoc(cast<MDNode>(unwrap(DL))), + unwrap(InsertAtEnd))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerator( + LLVMRustDIBuilderRef Builder, const char *Name, size_t NameLen, + int64_t Value, bool IsUnsigned) { + return wrap(Builder->createEnumerator(StringRef(Name, NameLen), Value, IsUnsigned)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, + uint32_t AlignInBits, LLVMMetadataRef Elements, + LLVMMetadataRef ClassTy, bool IsScoped) { + return wrap(Builder->createEnumerationType( + unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), + unwrapDI<DIFile>(File), LineNumber, + SizeInBits, AlignInBits, DINodeArray(unwrapDI<MDTuple>(Elements)), + unwrapDI<DIType>(ClassTy), "", IsScoped)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNumber, uint64_t SizeInBits, + uint32_t AlignInBits, LLVMRustDIFlags Flags, LLVMMetadataRef Elements, + unsigned RunTimeLang, const char *UniqueId, size_t UniqueIdLen) { + return wrap(Builder->createUnionType( + unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIFile>(File), + LineNumber, SizeInBits, AlignInBits, fromRust(Flags), + DINodeArray(unwrapDI<MDTuple>(Elements)), RunTimeLang, + StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, LLVMMetadataRef Ty) { + bool IsDefault = false; // FIXME: should we ever set this true? + return wrap(Builder->createTemplateTypeParameter( + unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), unwrapDI<DIType>(Ty), IsDefault)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateNameSpace( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, bool ExportSymbols) { + return wrap(Builder->createNameSpace( + unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen), ExportSymbols + )); +} + +extern "C" void +LLVMRustDICompositeTypeReplaceArrays(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef CompositeTy, + LLVMMetadataRef Elements, + LLVMMetadataRef Params) { + DICompositeType *Tmp = unwrapDI<DICompositeType>(CompositeTy); + Builder->replaceArrays(Tmp, DINodeArray(unwrap<MDTuple>(Elements)), + DINodeArray(unwrap<MDTuple>(Params))); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column, + LLVMMetadataRef ScopeRef, + LLVMMetadataRef InlinedAt) { + MDNode *Scope = unwrapDIPtr<MDNode>(ScopeRef); + DILocation *Loc = DILocation::get( + Scope->getContext(), Line, Column, Scope, + unwrapDIPtr<MDNode>(InlinedAt)); + return wrap(Loc); +} + +extern "C" uint64_t LLVMRustDIBuilderCreateOpDeref() { + return dwarf::DW_OP_deref; +} + +extern "C" uint64_t LLVMRustDIBuilderCreateOpPlusUconst() { + return dwarf::DW_OP_plus_uconst; +} + +extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) { + RawRustStringOstream OS(Str); + unwrap<llvm::Type>(Ty)->print(OS); +} + +extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, + RustStringRef Str) { + RawRustStringOstream OS(Str); + if (!V) { + OS << "(null)"; + } else { + OS << "("; + unwrap<llvm::Value>(V)->getType()->print(OS); + OS << ":"; + unwrap<llvm::Value>(V)->print(OS); + OS << ")"; + } +} + +// LLVMArrayType function does not support 64-bit ElementCount +extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy, + uint64_t ElementCount) { + return wrap(ArrayType::get(unwrap(ElementTy), ElementCount)); +} + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Twine, LLVMTwineRef) + +extern "C" void LLVMRustWriteTwineToString(LLVMTwineRef T, RustStringRef Str) { + RawRustStringOstream OS(Str); + unwrap(T)->print(OS); +} + +extern "C" void LLVMRustUnpackOptimizationDiagnostic( + LLVMDiagnosticInfoRef DI, RustStringRef PassNameOut, + LLVMValueRef *FunctionOut, unsigned* Line, unsigned* Column, + RustStringRef FilenameOut, RustStringRef MessageOut) { + // Undefined to call this not on an optimization diagnostic! + llvm::DiagnosticInfoOptimizationBase *Opt = + static_cast<llvm::DiagnosticInfoOptimizationBase *>(unwrap(DI)); + + RawRustStringOstream PassNameOS(PassNameOut); + PassNameOS << Opt->getPassName(); + *FunctionOut = wrap(&Opt->getFunction()); + + RawRustStringOstream FilenameOS(FilenameOut); + DiagnosticLocation loc = Opt->getLocation(); + if (loc.isValid()) { + *Line = loc.getLine(); + *Column = loc.getColumn(); + FilenameOS << loc.getAbsolutePath(); + } + + RawRustStringOstream MessageOS(MessageOut); + MessageOS << Opt->getMsg(); +} + +enum class LLVMRustDiagnosticLevel { + Error, + Warning, + Note, + Remark, +}; + +extern "C" void +LLVMRustUnpackInlineAsmDiagnostic(LLVMDiagnosticInfoRef DI, + LLVMRustDiagnosticLevel *LevelOut, + unsigned *CookieOut, + LLVMTwineRef *MessageOut) { + // Undefined to call this not on an inline assembly diagnostic! + llvm::DiagnosticInfoInlineAsm *IA = + static_cast<llvm::DiagnosticInfoInlineAsm *>(unwrap(DI)); + + *CookieOut = IA->getLocCookie(); + *MessageOut = wrap(&IA->getMsgStr()); + + switch (IA->getSeverity()) { + case DS_Error: + *LevelOut = LLVMRustDiagnosticLevel::Error; + break; + case DS_Warning: + *LevelOut = LLVMRustDiagnosticLevel::Warning; + break; + case DS_Note: + *LevelOut = LLVMRustDiagnosticLevel::Note; + break; + case DS_Remark: + *LevelOut = LLVMRustDiagnosticLevel::Remark; + break; + default: + report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); + } +} + +extern "C" void LLVMRustWriteDiagnosticInfoToString(LLVMDiagnosticInfoRef DI, + RustStringRef Str) { + RawRustStringOstream OS(Str); + DiagnosticPrinterRawOStream DP(OS); + unwrap(DI)->print(DP); +} + +enum class LLVMRustDiagnosticKind { + Other, + InlineAsm, + StackSize, + DebugMetadataVersion, + SampleProfile, + OptimizationRemark, + OptimizationRemarkMissed, + OptimizationRemarkAnalysis, + OptimizationRemarkAnalysisFPCommute, + OptimizationRemarkAnalysisAliasing, + OptimizationRemarkOther, + OptimizationFailure, + PGOProfile, + Linker, + Unsupported, + SrcMgr, +}; + +static LLVMRustDiagnosticKind toRust(DiagnosticKind Kind) { + switch (Kind) { + case DK_InlineAsm: + return LLVMRustDiagnosticKind::InlineAsm; + case DK_StackSize: + return LLVMRustDiagnosticKind::StackSize; + case DK_DebugMetadataVersion: + return LLVMRustDiagnosticKind::DebugMetadataVersion; + case DK_SampleProfile: + return LLVMRustDiagnosticKind::SampleProfile; + case DK_OptimizationRemark: + case DK_MachineOptimizationRemark: + return LLVMRustDiagnosticKind::OptimizationRemark; + case DK_OptimizationRemarkMissed: + case DK_MachineOptimizationRemarkMissed: + return LLVMRustDiagnosticKind::OptimizationRemarkMissed; + case DK_OptimizationRemarkAnalysis: + case DK_MachineOptimizationRemarkAnalysis: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysis; + case DK_OptimizationRemarkAnalysisFPCommute: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute; + case DK_OptimizationRemarkAnalysisAliasing: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing; + case DK_PGOProfile: + return LLVMRustDiagnosticKind::PGOProfile; + case DK_Linker: + return LLVMRustDiagnosticKind::Linker; + case DK_Unsupported: + return LLVMRustDiagnosticKind::Unsupported; +#if LLVM_VERSION_GE(13, 0) + case DK_SrcMgr: + return LLVMRustDiagnosticKind::SrcMgr; +#endif + default: + return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark) + ? LLVMRustDiagnosticKind::OptimizationRemarkOther + : LLVMRustDiagnosticKind::Other; + } +} + +extern "C" LLVMRustDiagnosticKind +LLVMRustGetDiagInfoKind(LLVMDiagnosticInfoRef DI) { + return toRust((DiagnosticKind)unwrap(DI)->getKind()); +} + +// This is kept distinct from LLVMGetTypeKind, because when +// a new type kind is added, the Rust-side enum must be +// updated or UB will result. +extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { + switch (unwrap(Ty)->getTypeID()) { + case Type::VoidTyID: + return LLVMVoidTypeKind; + case Type::HalfTyID: + return LLVMHalfTypeKind; + case Type::FloatTyID: + return LLVMFloatTypeKind; + case Type::DoubleTyID: + return LLVMDoubleTypeKind; + case Type::X86_FP80TyID: + return LLVMX86_FP80TypeKind; + case Type::FP128TyID: + return LLVMFP128TypeKind; + case Type::PPC_FP128TyID: + return LLVMPPC_FP128TypeKind; + case Type::LabelTyID: + return LLVMLabelTypeKind; + case Type::MetadataTyID: + return LLVMMetadataTypeKind; + case Type::IntegerTyID: + return LLVMIntegerTypeKind; + case Type::FunctionTyID: + return LLVMFunctionTypeKind; + case Type::StructTyID: + return LLVMStructTypeKind; + case Type::ArrayTyID: + return LLVMArrayTypeKind; + case Type::PointerTyID: + return LLVMPointerTypeKind; + case Type::FixedVectorTyID: + return LLVMVectorTypeKind; + case Type::X86_MMXTyID: + return LLVMX86_MMXTypeKind; + case Type::TokenTyID: + return LLVMTokenTypeKind; + case Type::ScalableVectorTyID: + return LLVMScalableVectorTypeKind; + case Type::BFloatTyID: + return LLVMBFloatTypeKind; + case Type::X86_AMXTyID: + return LLVMX86_AMXTypeKind; +#if LLVM_VERSION_GE(15, 0) && LLVM_VERSION_LT(16, 0) + case Type::DXILPointerTyID: + report_fatal_error("Rust does not support DirectX typed pointers."); + break; +#endif +#if LLVM_VERSION_GE(16, 0) + case Type::TypedPointerTyID: + report_fatal_error("Rust does not support typed pointers."); + break; +#endif + } + report_fatal_error("Unhandled TypeID."); +} + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) + +#if LLVM_VERSION_LT(13, 0) +using LLVMInlineAsmDiagHandlerTy = LLVMContext::InlineAsmDiagHandlerTy; +#else +using LLVMInlineAsmDiagHandlerTy = void*; +#endif + +extern "C" void LLVMRustSetInlineAsmDiagnosticHandler( + LLVMContextRef C, LLVMInlineAsmDiagHandlerTy H, void *CX) { + // Diagnostic handlers were unified in LLVM change 5de2d189e6ad, so starting + // with LLVM 13 this function is gone. +#if LLVM_VERSION_LT(13, 0) + unwrap(C)->setInlineAsmDiagnosticHandler(H, CX); +#endif +} + +extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic( + LLVMDiagnosticInfoRef DI, unsigned *Cookie) { +#if LLVM_VERSION_GE(13, 0) + llvm::DiagnosticInfoSrcMgr *SM = static_cast<llvm::DiagnosticInfoSrcMgr *>(unwrap(DI)); + *Cookie = SM->getLocCookie(); + return wrap(&SM->getSMDiag()); +#else + report_fatal_error("Shouldn't get called on older versions"); +#endif +} + +extern "C" bool LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, + RustStringRef MessageOut, + RustStringRef BufferOut, + LLVMRustDiagnosticLevel* LevelOut, + unsigned* LocOut, + unsigned* RangesOut, + size_t* NumRanges) { + SMDiagnostic& D = *unwrap(DRef); + RawRustStringOstream MessageOS(MessageOut); + MessageOS << D.getMessage(); + + switch (D.getKind()) { + case SourceMgr::DK_Error: + *LevelOut = LLVMRustDiagnosticLevel::Error; + break; + case SourceMgr::DK_Warning: + *LevelOut = LLVMRustDiagnosticLevel::Warning; + break; + case SourceMgr::DK_Note: + *LevelOut = LLVMRustDiagnosticLevel::Note; + break; + case SourceMgr::DK_Remark: + *LevelOut = LLVMRustDiagnosticLevel::Remark; + break; + default: + report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); + } + + if (D.getLoc() == SMLoc()) + return false; + + const SourceMgr &LSM = *D.getSourceMgr(); + const MemoryBuffer *LBuf = LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc())); + LLVMRustStringWriteImpl(BufferOut, LBuf->getBufferStart(), LBuf->getBufferSize()); + + *LocOut = D.getLoc().getPointer() - LBuf->getBufferStart(); + + *NumRanges = std::min(*NumRanges, D.getRanges().size()); + size_t LineStart = *LocOut - (size_t)D.getColumnNo(); + for (size_t i = 0; i < *NumRanges; i++) { + RangesOut[i * 2] = LineStart + D.getRanges()[i].first; + RangesOut[i * 2 + 1] = LineStart + D.getRanges()[i].second; + } + + return true; +} + +extern "C" LLVMValueRef LLVMRustBuildCleanupPad(LLVMBuilderRef B, + LLVMValueRef ParentPad, + unsigned ArgCount, + LLVMValueRef *LLArgs, + const char *Name) { + Value **Args = unwrap(LLArgs); + if (ParentPad == nullptr) { + Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); + ParentPad = wrap(Constant::getNullValue(Ty)); + } + return wrap(unwrap(B)->CreateCleanupPad( + unwrap(ParentPad), ArrayRef<Value *>(Args, ArgCount), Name)); +} + +extern "C" LLVMValueRef LLVMRustBuildCleanupRet(LLVMBuilderRef B, + LLVMValueRef CleanupPad, + LLVMBasicBlockRef UnwindBB) { + CleanupPadInst *Inst = cast<CleanupPadInst>(unwrap(CleanupPad)); + return wrap(unwrap(B)->CreateCleanupRet(Inst, unwrap(UnwindBB))); +} + +extern "C" LLVMValueRef +LLVMRustBuildCatchPad(LLVMBuilderRef B, LLVMValueRef ParentPad, + unsigned ArgCount, LLVMValueRef *LLArgs, const char *Name) { + Value **Args = unwrap(LLArgs); + return wrap(unwrap(B)->CreateCatchPad( + unwrap(ParentPad), ArrayRef<Value *>(Args, ArgCount), Name)); +} + +extern "C" LLVMValueRef LLVMRustBuildCatchRet(LLVMBuilderRef B, + LLVMValueRef Pad, + LLVMBasicBlockRef BB) { + return wrap(unwrap(B)->CreateCatchRet(cast<CatchPadInst>(unwrap(Pad)), + unwrap(BB))); +} + +extern "C" LLVMValueRef LLVMRustBuildCatchSwitch(LLVMBuilderRef B, + LLVMValueRef ParentPad, + LLVMBasicBlockRef BB, + unsigned NumHandlers, + const char *Name) { + if (ParentPad == nullptr) { + Type *Ty = Type::getTokenTy(unwrap(B)->getContext()); + ParentPad = wrap(Constant::getNullValue(Ty)); + } + return wrap(unwrap(B)->CreateCatchSwitch(unwrap(ParentPad), unwrap(BB), + NumHandlers, Name)); +} + +extern "C" void LLVMRustAddHandler(LLVMValueRef CatchSwitchRef, + LLVMBasicBlockRef Handler) { + Value *CatchSwitch = unwrap(CatchSwitchRef); + cast<CatchSwitchInst>(CatchSwitch)->addHandler(unwrap(Handler)); +} + +extern "C" OperandBundleDef *LLVMRustBuildOperandBundleDef(const char *Name, + LLVMValueRef *Inputs, + unsigned NumInputs) { + return new OperandBundleDef(Name, makeArrayRef(unwrap(Inputs), NumInputs)); +} + +extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) { + delete Bundle; +} + +extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + OperandBundleDef *Bundle) { + Value *Callee = unwrap(Fn); + FunctionType *FTy = unwrap<FunctionType>(Ty); + unsigned Len = Bundle ? 1 : 0; + ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len); + return wrap(unwrap(B)->CreateCall( + FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles)); +} + +extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) { + return wrap(llvm::Intrinsic::getDeclaration(unwrap(M), + (llvm::Intrinsic::ID)llvm::Intrinsic::instrprof_increment)); +} + +extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size, bool IsVolatile) { + return wrap(unwrap(B)->CreateMemCpy( + unwrap(Dst), MaybeAlign(DstAlign), + unwrap(Src), MaybeAlign(SrcAlign), + unwrap(Size), IsVolatile)); +} + +extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, + LLVMValueRef Size, bool IsVolatile) { + return wrap(unwrap(B)->CreateMemMove( + unwrap(Dst), MaybeAlign(DstAlign), + unwrap(Src), MaybeAlign(SrcAlign), + unwrap(Size), IsVolatile)); +} + +extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, + LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Val, + LLVMValueRef Size, bool IsVolatile) { + return wrap(unwrap(B)->CreateMemSet( + unwrap(Dst), unwrap(Val), unwrap(Size), MaybeAlign(DstAlign), IsVolatile)); +} + +extern "C" LLVMValueRef +LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, + OperandBundleDef *Bundle, const char *Name) { + Value *Callee = unwrap(Fn); + FunctionType *FTy = unwrap<FunctionType>(Ty); + unsigned Len = Bundle ? 1 : 0; + ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len); + return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch), + makeArrayRef(unwrap(Args), NumArgs), + Bundles, Name)); +} + +extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, + LLVMBasicBlockRef BB) { + auto Point = unwrap(BB)->getFirstInsertionPt(); + unwrap(B)->SetInsertPoint(unwrap(BB), Point); +} + +extern "C" void LLVMRustSetComdat(LLVMModuleRef M, LLVMValueRef V, + const char *Name, size_t NameLen) { + Triple TargetTriple(unwrap(M)->getTargetTriple()); + GlobalObject *GV = unwrap<GlobalObject>(V); + if (TargetTriple.supportsCOMDAT()) { + StringRef NameRef(Name, NameLen); + GV->setComdat(unwrap(M)->getOrInsertComdat(NameRef)); + } +} + +enum class LLVMRustLinkage { + ExternalLinkage = 0, + AvailableExternallyLinkage = 1, + LinkOnceAnyLinkage = 2, + LinkOnceODRLinkage = 3, + WeakAnyLinkage = 4, + WeakODRLinkage = 5, + AppendingLinkage = 6, + InternalLinkage = 7, + PrivateLinkage = 8, + ExternalWeakLinkage = 9, + CommonLinkage = 10, +}; + +static LLVMRustLinkage toRust(LLVMLinkage Linkage) { + switch (Linkage) { + case LLVMExternalLinkage: + return LLVMRustLinkage::ExternalLinkage; + case LLVMAvailableExternallyLinkage: + return LLVMRustLinkage::AvailableExternallyLinkage; + case LLVMLinkOnceAnyLinkage: + return LLVMRustLinkage::LinkOnceAnyLinkage; + case LLVMLinkOnceODRLinkage: + return LLVMRustLinkage::LinkOnceODRLinkage; + case LLVMWeakAnyLinkage: + return LLVMRustLinkage::WeakAnyLinkage; + case LLVMWeakODRLinkage: + return LLVMRustLinkage::WeakODRLinkage; + case LLVMAppendingLinkage: + return LLVMRustLinkage::AppendingLinkage; + case LLVMInternalLinkage: + return LLVMRustLinkage::InternalLinkage; + case LLVMPrivateLinkage: + return LLVMRustLinkage::PrivateLinkage; + case LLVMExternalWeakLinkage: + return LLVMRustLinkage::ExternalWeakLinkage; + case LLVMCommonLinkage: + return LLVMRustLinkage::CommonLinkage; + default: + report_fatal_error("Invalid LLVMRustLinkage value!"); + } +} + +static LLVMLinkage fromRust(LLVMRustLinkage Linkage) { + switch (Linkage) { + case LLVMRustLinkage::ExternalLinkage: + return LLVMExternalLinkage; + case LLVMRustLinkage::AvailableExternallyLinkage: + return LLVMAvailableExternallyLinkage; + case LLVMRustLinkage::LinkOnceAnyLinkage: + return LLVMLinkOnceAnyLinkage; + case LLVMRustLinkage::LinkOnceODRLinkage: + return LLVMLinkOnceODRLinkage; + case LLVMRustLinkage::WeakAnyLinkage: + return LLVMWeakAnyLinkage; + case LLVMRustLinkage::WeakODRLinkage: + return LLVMWeakODRLinkage; + case LLVMRustLinkage::AppendingLinkage: + return LLVMAppendingLinkage; + case LLVMRustLinkage::InternalLinkage: + return LLVMInternalLinkage; + case LLVMRustLinkage::PrivateLinkage: + return LLVMPrivateLinkage; + case LLVMRustLinkage::ExternalWeakLinkage: + return LLVMExternalWeakLinkage; + case LLVMRustLinkage::CommonLinkage: + return LLVMCommonLinkage; + } + report_fatal_error("Invalid LLVMRustLinkage value!"); +} + +extern "C" LLVMRustLinkage LLVMRustGetLinkage(LLVMValueRef V) { + return toRust(LLVMGetLinkage(V)); +} + +extern "C" void LLVMRustSetLinkage(LLVMValueRef V, + LLVMRustLinkage RustLinkage) { + LLVMSetLinkage(V, fromRust(RustLinkage)); +} + +extern "C" LLVMValueRef LLVMRustConstInBoundsGEP2(LLVMTypeRef Ty, + LLVMValueRef ConstantVal, + LLVMValueRef *ConstantIndices, + unsigned NumIndices) { + ArrayRef<Constant *> IdxList(unwrap<Constant>(ConstantIndices, NumIndices), + NumIndices); + Constant *Val = unwrap<Constant>(ConstantVal); + return wrap(ConstantExpr::getInBoundsGetElementPtr(unwrap(Ty), Val, IdxList)); +} + +// Returns true if both high and low were successfully set. Fails in case constant wasn’t any of +// the common sizes (1, 8, 16, 32, 64, 128 bits) +extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, uint64_t *high, uint64_t *low) +{ + auto C = unwrap<llvm::ConstantInt>(CV); + if (C->getBitWidth() > 128) { return false; } + APInt AP; +#if LLVM_VERSION_GE(15, 0) + if (sext) { + AP = C->getValue().sext(128); + } else { + AP = C->getValue().zext(128); + } +#else + if (sext) { + AP = C->getValue().sextOrSelf(128); + } else { + AP = C->getValue().zextOrSelf(128); + } +#endif + *low = AP.getLoBits(64).getZExtValue(); + *high = AP.getHiBits(64).getZExtValue(); + return true; +} + +enum class LLVMRustVisibility { + Default = 0, + Hidden = 1, + Protected = 2, +}; + +static LLVMRustVisibility toRust(LLVMVisibility Vis) { + switch (Vis) { + case LLVMDefaultVisibility: + return LLVMRustVisibility::Default; + case LLVMHiddenVisibility: + return LLVMRustVisibility::Hidden; + case LLVMProtectedVisibility: + return LLVMRustVisibility::Protected; + } + report_fatal_error("Invalid LLVMRustVisibility value!"); +} + +static LLVMVisibility fromRust(LLVMRustVisibility Vis) { + switch (Vis) { + case LLVMRustVisibility::Default: + return LLVMDefaultVisibility; + case LLVMRustVisibility::Hidden: + return LLVMHiddenVisibility; + case LLVMRustVisibility::Protected: + return LLVMProtectedVisibility; + } + report_fatal_error("Invalid LLVMRustVisibility value!"); +} + +extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) { + return toRust(LLVMGetVisibility(V)); +} + +// Oh hey, a binding that makes sense for once? (because LLVM’s own do not) +extern "C" LLVMValueRef LLVMRustBuildIntCast(LLVMBuilderRef B, LLVMValueRef Val, + LLVMTypeRef DestTy, bool isSigned) { + return wrap(unwrap(B)->CreateIntCast(unwrap(Val), unwrap(DestTy), isSigned, "")); +} + +extern "C" void LLVMRustSetVisibility(LLVMValueRef V, + LLVMRustVisibility RustVisibility) { + LLVMSetVisibility(V, fromRust(RustVisibility)); +} + +extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) { + unwrap<GlobalValue>(Global)->setDSOLocal(is_dso_local); +} + +struct LLVMRustModuleBuffer { + std::string data; +}; + +extern "C" LLVMRustModuleBuffer* +LLVMRustModuleBufferCreate(LLVMModuleRef M) { + auto Ret = std::make_unique<LLVMRustModuleBuffer>(); + { + raw_string_ostream OS(Ret->data); + { + legacy::PassManager PM; + PM.add(createBitcodeWriterPass(OS)); + PM.run(*unwrap(M)); + } + } + return Ret.release(); +} + +extern "C" void +LLVMRustModuleBufferFree(LLVMRustModuleBuffer *Buffer) { + delete Buffer; +} + +extern "C" const void* +LLVMRustModuleBufferPtr(const LLVMRustModuleBuffer *Buffer) { + return Buffer->data.data(); +} + +extern "C" size_t +LLVMRustModuleBufferLen(const LLVMRustModuleBuffer *Buffer) { + return Buffer->data.length(); +} + +extern "C" uint64_t +LLVMRustModuleCost(LLVMModuleRef M) { + auto f = unwrap(M)->functions(); + return std::distance(std::begin(f), std::end(f)); +} + +// Vector reductions: +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateFAddReduce(unwrap(Acc),unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFMul(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateFMulReduce(unwrap(Acc),unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceAdd(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateAddReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceMul(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateMulReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceAnd(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateAndReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceOr(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateOrReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceXor(LLVMBuilderRef B, LLVMValueRef Src) { + return wrap(unwrap(B)->CreateXorReduce(unwrap(Src))); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceMin(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { + return wrap(unwrap(B)->CreateIntMinReduce(unwrap(Src), IsSigned)); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceMax(LLVMBuilderRef B, LLVMValueRef Src, bool IsSigned) { + return wrap(unwrap(B)->CreateIntMaxReduce(unwrap(Src), IsSigned)); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFMin(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { + Instruction *I = unwrap(B)->CreateFPMinReduce(unwrap(Src)); + I->setHasNoNaNs(NoNaN); + return wrap(I); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { + Instruction *I = unwrap(B)->CreateFPMaxReduce(unwrap(Src)); + I->setHasNoNaNs(NoNaN); + return wrap(I); +} + +extern "C" LLVMValueRef +LLVMRustBuildMinNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { + return wrap(unwrap(B)->CreateMinNum(unwrap(LHS),unwrap(RHS))); +} +extern "C" LLVMValueRef +LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS) { + return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS),unwrap(RHS))); +} + +// This struct contains all necessary info about a symbol exported from a DLL. +struct LLVMRustCOFFShortExport { + const char* name; + bool ordinal_present; + // The value of `ordinal` is only meaningful if `ordinal_present` is true. + uint16_t ordinal; +}; + +// Machine must be a COFF machine type, as defined in PE specs. +extern "C" LLVMRustResult LLVMRustWriteImportLibrary( + const char* ImportName, + const char* Path, + const LLVMRustCOFFShortExport* Exports, + size_t NumExports, + uint16_t Machine, + bool MinGW) +{ + std::vector<llvm::object::COFFShortExport> ConvertedExports; + ConvertedExports.reserve(NumExports); + + for (size_t i = 0; i < NumExports; ++i) { + bool ordinal_present = Exports[i].ordinal_present; + uint16_t ordinal = ordinal_present ? Exports[i].ordinal : 0; + ConvertedExports.push_back(llvm::object::COFFShortExport{ + Exports[i].name, // Name + std::string{}, // ExtName + std::string{}, // SymbolName + std::string{}, // AliasTarget + ordinal, // Ordinal + ordinal_present, // Noname + false, // Data + false, // Private + false // Constant + }); + } + + auto Error = llvm::object::writeImportLibrary( + ImportName, + Path, + ConvertedExports, + static_cast<llvm::COFF::MachineTypes>(Machine), + MinGW); + if (Error) { + std::string errorString; + llvm::raw_string_ostream stream(errorString); + stream << Error; + stream.flush(); + LLVMRustSetLastError(errorString.c_str()); + return LLVMRustResult::Failure; + } else { + return LLVMRustResult::Success; + } +} + +// Transfers ownership of DiagnosticHandler unique_ptr to the caller. +extern "C" DiagnosticHandler * +LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) { + std::unique_ptr<DiagnosticHandler> DH = unwrap(C)->getDiagnosticHandler(); + return DH.release(); +} + +// Sets unique_ptr to object of DiagnosticHandler to provide custom diagnostic +// handling. Ownership of the handler is moved to the LLVMContext. +extern "C" void LLVMRustContextSetDiagnosticHandler(LLVMContextRef C, + DiagnosticHandler *DH) { + unwrap(C)->setDiagnosticHandler(std::unique_ptr<DiagnosticHandler>(DH)); +} + +using LLVMDiagnosticHandlerTy = DiagnosticHandler::DiagnosticHandlerTy; + +// Configures a diagnostic handler that invokes provided callback when a +// backend needs to emit a diagnostic. +// +// When RemarkAllPasses is true, remarks are enabled for all passes. Otherwise +// the RemarkPasses array specifies individual passes for which remarks will be +// enabled. +extern "C" void LLVMRustContextConfigureDiagnosticHandler( + LLVMContextRef C, LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, + void *DiagnosticHandlerContext, bool RemarkAllPasses, + const char * const * RemarkPasses, size_t RemarkPassesLen) { + + class RustDiagnosticHandler final : public DiagnosticHandler { + public: + RustDiagnosticHandler(LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, + void *DiagnosticHandlerContext, + bool RemarkAllPasses, + std::vector<std::string> RemarkPasses) + : DiagnosticHandlerCallback(DiagnosticHandlerCallback), + DiagnosticHandlerContext(DiagnosticHandlerContext), + RemarkAllPasses(RemarkAllPasses), + RemarkPasses(RemarkPasses) {} + + virtual bool handleDiagnostics(const DiagnosticInfo &DI) override { + if (DiagnosticHandlerCallback) { + DiagnosticHandlerCallback(DI, DiagnosticHandlerContext); + return true; + } + return false; + } + + bool isAnalysisRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isMissedOptRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isPassedOptRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isAnyRemarkEnabled() const override { + return RemarkAllPasses || !RemarkPasses.empty(); + } + + private: + bool isRemarkEnabled(StringRef PassName) const { + if (RemarkAllPasses) + return true; + + for (auto &Pass : RemarkPasses) + if (Pass == PassName) + return true; + + return false; + } + + LLVMDiagnosticHandlerTy DiagnosticHandlerCallback = nullptr; + void *DiagnosticHandlerContext = nullptr; + + bool RemarkAllPasses = false; + std::vector<std::string> RemarkPasses; + }; + + std::vector<std::string> Passes; + for (size_t I = 0; I != RemarkPassesLen; ++I) + Passes.push_back(RemarkPasses[I]); + + unwrap(C)->setDiagnosticHandler(std::make_unique<RustDiagnosticHandler>( + DiagnosticHandlerCallback, DiagnosticHandlerContext, RemarkAllPasses, Passes)); +} + +extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { + RawRustStringOstream OS(Str); + GlobalValue *GV = unwrap<GlobalValue>(V); + Mangler().getNameWithPrefix(OS, GV, true); +} + +// LLVMGetAggregateElement was added in LLVM 15. For earlier LLVM versions just +// use its implementation. +#if LLVM_VERSION_LT(15, 0) +extern "C" LLVMValueRef LLVMGetAggregateElement(LLVMValueRef C, unsigned Idx) { + return wrap(unwrap<Constant>(C)->getAggregateElement(Idx)); +} +#endif + +extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { +#if LLVM_VERSION_GE(15, 0) + auto *CB = unwrap<CallBase>(CallSite); + switch (CB->getIntrinsicID()) { + case Intrinsic::arm_ldrex: + return 0; + case Intrinsic::arm_strex: + return 1; + } +#endif + return -1; +} diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs new file mode 100644 index 000000000..8eade02a4 --- /dev/null +++ b/compiler/rustc_llvm/src/lib.rs @@ -0,0 +1,188 @@ +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] + +// NOTE: This crate only exists to allow linking on mingw targets. + +use libc::{c_char, size_t}; +use std::cell::RefCell; +use std::slice; + +#[repr(C)] +pub struct RustString { + pub bytes: RefCell<Vec<u8>>, +} + +impl RustString { + pub fn len(&self) -> usize { + self.bytes.borrow().len() + } + + pub fn is_empty(&self) -> bool { + self.bytes.borrow().is_empty() + } +} + +/// Appending to a Rust string -- used by RawRustStringOstream. +#[no_mangle] +pub unsafe extern "C" fn LLVMRustStringWriteImpl( + sr: &RustString, + ptr: *const c_char, + size: size_t, +) { + let slice = slice::from_raw_parts(ptr as *const u8, size as usize); + + sr.bytes.borrow_mut().extend_from_slice(slice); +} + +/// Initialize targets enabled by the build script via `cfg(llvm_component = "...")`. +/// N.B., this function can't be moved to `rustc_codegen_llvm` because of the `cfg`s. +pub fn initialize_available_targets() { + macro_rules! init_target( + ($cfg:meta, $($method:ident),*) => { { + #[cfg($cfg)] + fn init() { + extern "C" { + $(fn $method();)* + } + unsafe { + $($method();)* + } + } + #[cfg(not($cfg))] + fn init() { } + init(); + } } + ); + init_target!( + llvm_component = "x86", + LLVMInitializeX86TargetInfo, + LLVMInitializeX86Target, + LLVMInitializeX86TargetMC, + LLVMInitializeX86AsmPrinter, + LLVMInitializeX86AsmParser + ); + init_target!( + llvm_component = "arm", + LLVMInitializeARMTargetInfo, + LLVMInitializeARMTarget, + LLVMInitializeARMTargetMC, + LLVMInitializeARMAsmPrinter, + LLVMInitializeARMAsmParser + ); + init_target!( + llvm_component = "aarch64", + LLVMInitializeAArch64TargetInfo, + LLVMInitializeAArch64Target, + LLVMInitializeAArch64TargetMC, + LLVMInitializeAArch64AsmPrinter, + LLVMInitializeAArch64AsmParser + ); + init_target!( + llvm_component = "amdgpu", + LLVMInitializeAMDGPUTargetInfo, + LLVMInitializeAMDGPUTarget, + LLVMInitializeAMDGPUTargetMC, + LLVMInitializeAMDGPUAsmPrinter, + LLVMInitializeAMDGPUAsmParser + ); + init_target!( + llvm_component = "avr", + LLVMInitializeAVRTargetInfo, + LLVMInitializeAVRTarget, + LLVMInitializeAVRTargetMC, + LLVMInitializeAVRAsmPrinter, + LLVMInitializeAVRAsmParser + ); + init_target!( + llvm_component = "m68k", + LLVMInitializeM68kTargetInfo, + LLVMInitializeM68kTarget, + LLVMInitializeM68kTargetMC, + LLVMInitializeM68kAsmPrinter, + LLVMInitializeM68kAsmParser + ); + init_target!( + llvm_component = "mips", + LLVMInitializeMipsTargetInfo, + LLVMInitializeMipsTarget, + LLVMInitializeMipsTargetMC, + LLVMInitializeMipsAsmPrinter, + LLVMInitializeMipsAsmParser + ); + init_target!( + llvm_component = "powerpc", + LLVMInitializePowerPCTargetInfo, + LLVMInitializePowerPCTarget, + LLVMInitializePowerPCTargetMC, + LLVMInitializePowerPCAsmPrinter, + LLVMInitializePowerPCAsmParser + ); + init_target!( + llvm_component = "systemz", + LLVMInitializeSystemZTargetInfo, + LLVMInitializeSystemZTarget, + LLVMInitializeSystemZTargetMC, + LLVMInitializeSystemZAsmPrinter, + LLVMInitializeSystemZAsmParser + ); + init_target!( + llvm_component = "jsbackend", + LLVMInitializeJSBackendTargetInfo, + LLVMInitializeJSBackendTarget, + LLVMInitializeJSBackendTargetMC + ); + init_target!( + llvm_component = "msp430", + LLVMInitializeMSP430TargetInfo, + LLVMInitializeMSP430Target, + LLVMInitializeMSP430TargetMC, + LLVMInitializeMSP430AsmPrinter, + LLVMInitializeMSP430AsmParser + ); + init_target!( + llvm_component = "riscv", + LLVMInitializeRISCVTargetInfo, + LLVMInitializeRISCVTarget, + LLVMInitializeRISCVTargetMC, + LLVMInitializeRISCVAsmPrinter, + LLVMInitializeRISCVAsmParser + ); + init_target!( + llvm_component = "sparc", + LLVMInitializeSparcTargetInfo, + LLVMInitializeSparcTarget, + LLVMInitializeSparcTargetMC, + LLVMInitializeSparcAsmPrinter, + LLVMInitializeSparcAsmParser + ); + init_target!( + llvm_component = "nvptx", + LLVMInitializeNVPTXTargetInfo, + LLVMInitializeNVPTXTarget, + LLVMInitializeNVPTXTargetMC, + LLVMInitializeNVPTXAsmPrinter + ); + init_target!( + llvm_component = "hexagon", + LLVMInitializeHexagonTargetInfo, + LLVMInitializeHexagonTarget, + LLVMInitializeHexagonTargetMC, + LLVMInitializeHexagonAsmPrinter, + LLVMInitializeHexagonAsmParser + ); + init_target!( + llvm_component = "webassembly", + LLVMInitializeWebAssemblyTargetInfo, + LLVMInitializeWebAssemblyTarget, + LLVMInitializeWebAssemblyTargetMC, + LLVMInitializeWebAssemblyAsmPrinter, + LLVMInitializeWebAssemblyAsmParser + ); + init_target!( + llvm_component = "bpf", + LLVMInitializeBPFTargetInfo, + LLVMInitializeBPFTarget, + LLVMInitializeBPFTargetMC, + LLVMInitializeBPFAsmPrinter, + LLVMInitializeBPFAsmParser + ); +} |