diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-18 02:49:42 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-18 02:49:42 +0000 |
commit | 837b550238aa671a591ccf282dddeab29cadb206 (patch) | |
tree | 914b6b8862bace72bd3245ca184d374b08d8a672 /src/bootstrap/bootstrap.py | |
parent | Adding debian version 1.70.0+dfsg2-1. (diff) | |
download | rustc-837b550238aa671a591ccf282dddeab29cadb206.tar.xz rustc-837b550238aa671a591ccf282dddeab29cadb206.zip |
Merging upstream version 1.71.1+dfsg1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/bootstrap/bootstrap.py')
-rw-r--r-- | src/bootstrap/bootstrap.py | 350 |
1 files changed, 240 insertions, 110 deletions
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 025145244..58d1926ad 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -13,17 +13,35 @@ import tarfile import tempfile from time import time +from multiprocessing import Pool, cpu_count try: import lzma except ImportError: lzma = None -if sys.platform == 'win32': +def platform_is_win32(): + return sys.platform == 'win32' + +if platform_is_win32(): EXE_SUFFIX = ".exe" else: EXE_SUFFIX = "" +def get_cpus(): + if hasattr(os, "sched_getaffinity"): + return len(os.sched_getaffinity(0)) + if hasattr(os, "cpu_count"): + cpus = os.cpu_count() + if cpus is not None: + return cpus + try: + return cpu_count() + except NotImplementedError: + return 1 + + + def get(base, url, path, checksums, verbose=False): with tempfile.NamedTemporaryFile(delete=False) as temp_file: temp_path = temp_file.name @@ -39,23 +57,23 @@ def get(base, url, path, checksums, verbose=False): if os.path.exists(path): if verify(path, sha256, False): if verbose: - print("using already-download file", path) + print("using already-download file", path, file=sys.stderr) return else: if verbose: print("ignoring already-download file", - path, "due to failed verification") + path, "due to failed verification", file=sys.stderr) os.unlink(path) download(temp_path, "{}/{}".format(base, url), True, verbose) if not verify(temp_path, sha256, verbose): raise RuntimeError("failed verification") if verbose: - print("moving {} to {}".format(temp_path, path)) + print("moving {} to {}".format(temp_path, path), file=sys.stderr) shutil.move(temp_path, path) finally: if os.path.isfile(temp_path): if verbose: - print("removing", temp_path) + print("removing", temp_path, file=sys.stderr) os.unlink(temp_path) @@ -65,7 +83,7 @@ def download(path, url, probably_big, verbose): _download(path, url, probably_big, verbose, True) return except RuntimeError: - print("\nspurious failure, trying again") + print("\nspurious failure, trying again", file=sys.stderr) _download(path, url, probably_big, verbose, False) @@ -76,9 +94,8 @@ def _download(path, url, probably_big, verbose, exception): # - If we are on win32 fallback to powershell # - Otherwise raise the error if appropriate if probably_big or verbose: - print("downloading {}".format(url)) + print("downloading {}".format(url), file=sys.stderr) - platform_is_win32 = sys.platform == 'win32' try: if probably_big or verbose: option = "-#" @@ -86,21 +103,21 @@ def _download(path, url, probably_big, verbose, exception): option = "-s" # If curl is not present on Win32, we should not sys.exit # but raise `CalledProcessError` or `OSError` instead - require(["curl", "--version"], exception=platform_is_win32) + require(["curl", "--version"], exception=platform_is_win32()) with open(path, "wb") as outfile: run(["curl", option, "-L", # Follow redirect. "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds "--connect-timeout", "30", # timeout if cannot connect within 30 seconds - "--retry", "3", "-Sf", url], + "--retry", "3", "-SRf", url], stdout=outfile, #Implements cli redirect operator '>' verbose=verbose, exception=True, # Will raise RuntimeError on failure ) except (subprocess.CalledProcessError, OSError, RuntimeError): # see http://serverfault.com/questions/301128/how-to-download - if platform_is_win32: - run(["PowerShell.exe", "/nologo", "-Command", + if platform_is_win32(): + run_powershell([ "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], verbose=verbose, @@ -113,20 +130,20 @@ def _download(path, url, probably_big, verbose, exception): def verify(path, expected, verbose): """Check if the sha256 sum of the given path is valid""" if verbose: - print("verifying", path) + print("verifying", path, file=sys.stderr) with open(path, "rb") as source: found = hashlib.sha256(source.read()).hexdigest() verified = found == expected if not verified: print("invalid checksum:\n" " found: {}\n" - " expected: {}".format(found, expected)) + " expected: {}".format(found, expected), file=sys.stderr) return verified def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): """Unpack the given tarball file""" - print("extracting", tarball) + print("extracting", tarball, file=sys.stderr) fname = os.path.basename(tarball).replace(tarball_suffix, "") with contextlib.closing(tarfile.open(tarball)) as tar: for member in tar.getnames(): @@ -139,7 +156,7 @@ def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): dst_path = os.path.join(dst, name) if verbose: - print(" extracting", member) + print(" extracting", member, file=sys.stderr) tar.extract(member, dst) src_path = os.path.join(dst, member) if os.path.isdir(src_path) and os.path.exists(dst_path): @@ -151,7 +168,7 @@ def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs): """Run a child program in a new process""" if verbose: - print("running: " + ' '.join(args)) + print("running: " + ' '.join(args), file=sys.stderr) sys.stdout.flush() # Ensure that the .exe is used on Windows just in case a Linux ELF has been # compiled in the same directory. @@ -174,6 +191,10 @@ def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs): else: sys.exit(err) +def run_powershell(script, *args, **kwargs): + """Run a powershell script""" + run(["PowerShell.exe", "/nologo", "-Command"] + script, *args, **kwargs) + def require(cmd, exit=True, exception=False): '''Run a command, returning its output. @@ -187,8 +208,8 @@ def require(cmd, exit=True, exception=False): if exception: raise elif exit: - print("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) - print("Please make sure it's installed and in the path.") + print("error: unable to run `{}`: {}".format(' '.join(cmd), exc), file=sys.stderr) + print("Please make sure it's installed and in the path.", file=sys.stderr) sys.exit(1) return None @@ -205,38 +226,41 @@ def format_build_time(duration): def default_build_triple(verbose): """Build triple as in LLVM""" - # If the user already has a host build triple with an existing `rustc` - # install, use their preference. This fixes most issues with Windows builds - # being detected as GNU instead of MSVC. + # If we're on Windows and have an existing `rustc` toolchain, use `rustc --version --verbose` + # to find our host target triple. This fixes an issue with Windows builds being detected + # as GNU instead of MSVC. + # Otherwise, detect it via `uname` default_encoding = sys.getdefaultencoding() - try: - version = subprocess.check_output(["rustc", "--version", "--verbose"], - stderr=subprocess.DEVNULL) - version = version.decode(default_encoding) - host = next(x for x in version.split('\n') if x.startswith("host: ")) - triple = host.split("host: ")[1] - if verbose: - print("detected default triple {} from pre-installed rustc".format(triple)) - return triple - except Exception as e: - if verbose: - print("pre-installed rustc not detected: {}".format(e)) - print("falling back to auto-detect") - required = sys.platform != 'win32' - ostype = require(["uname", "-s"], exit=required) - cputype = require(['uname', '-m'], exit=required) + if platform_is_win32(): + try: + version = subprocess.check_output(["rustc", "--version", "--verbose"], + stderr=subprocess.DEVNULL) + version = version.decode(default_encoding) + host = next(x for x in version.split('\n') if x.startswith("host: ")) + triple = host.split("host: ")[1] + if verbose: + print("detected default triple {} from pre-installed rustc".format(triple), + file=sys.stderr) + return triple + except Exception as e: + if verbose: + print("pre-installed rustc not detected: {}".format(e), + file=sys.stderr) + print("falling back to auto-detect", file=sys.stderr) + + required = not platform_is_win32() + uname = require(["uname", "-smp"], exit=required) # If we do not have `uname`, assume Windows. - if ostype is None or cputype is None: + if uname is None: return 'x86_64-pc-windows-msvc' - ostype = ostype.decode(default_encoding) - cputype = cputype.decode(default_encoding) + kernel, cputype, processor = uname.decode(default_encoding).split() # The goal here is to come up with the same triple as LLVM would, # at least for the subset of platforms we're willing to target. - ostype_mapper = { + kerneltype_mapper = { 'Darwin': 'apple-darwin', 'DragonFly': 'unknown-dragonfly', 'FreeBSD': 'unknown-freebsd', @@ -246,17 +270,18 @@ def default_build_triple(verbose): } # Consider the direct transformation first and then the special cases - if ostype in ostype_mapper: - ostype = ostype_mapper[ostype] - elif ostype == 'Linux': - os_from_sp = subprocess.check_output( - ['uname', '-o']).strip().decode(default_encoding) - if os_from_sp == 'Android': - ostype = 'linux-android' + if kernel in kerneltype_mapper: + kernel = kerneltype_mapper[kernel] + elif kernel == 'Linux': + # Apple doesn't support `-o` so this can't be used in the combined + # uname invocation above + ostype = require(["uname", "-o"], exit=required).decode(default_encoding) + if ostype == 'Android': + kernel = 'linux-android' else: - ostype = 'unknown-linux-gnu' - elif ostype == 'SunOS': - ostype = 'pc-solaris' + kernel = 'unknown-linux-gnu' + elif kernel == 'SunOS': + kernel = 'pc-solaris' # On Solaris, uname -m will return a machine classification instead # of a cpu type, so uname -p is recommended instead. However, the # output from that option is too generic for our purposes (it will @@ -265,34 +290,34 @@ def default_build_triple(verbose): cputype = require(['isainfo', '-k']).decode(default_encoding) # sparc cpus have sun as a target vendor if 'sparc' in cputype: - ostype = 'sun-solaris' - elif ostype.startswith('MINGW'): + kernel = 'sun-solaris' + elif kernel.startswith('MINGW'): # msys' `uname` does not print gcc configuration, but prints msys # configuration. so we cannot believe `uname -m`: # msys1 is always i686 and msys2 is always x86_64. # instead, msys defines $MSYSTEM which is MINGW32 on i686 and # MINGW64 on x86_64. - ostype = 'pc-windows-gnu' + kernel = 'pc-windows-gnu' cputype = 'i686' if os.environ.get('MSYSTEM') == 'MINGW64': cputype = 'x86_64' - elif ostype.startswith('MSYS'): - ostype = 'pc-windows-gnu' - elif ostype.startswith('CYGWIN_NT'): + elif kernel.startswith('MSYS'): + kernel = 'pc-windows-gnu' + elif kernel.startswith('CYGWIN_NT'): cputype = 'i686' - if ostype.endswith('WOW64'): + if kernel.endswith('WOW64'): cputype = 'x86_64' - ostype = 'pc-windows-gnu' - elif sys.platform == 'win32': + kernel = 'pc-windows-gnu' + elif platform_is_win32(): # Some Windows platforms might have a `uname` command that returns a # non-standard string (e.g. gnuwin32 tools returns `windows32`). In # these cases, fall back to using sys.platform. return 'x86_64-pc-windows-msvc' else: - err = "unknown OS type: {}".format(ostype) + err = "unknown OS type: {}".format(kernel) sys.exit(err) - if cputype in ['powerpc', 'riscv'] and ostype == 'unknown-freebsd': + if cputype in ['powerpc', 'riscv'] and kernel == 'unknown-freebsd': cputype = subprocess.check_output( ['uname', '-p']).strip().decode(default_encoding) cputype_mapper = { @@ -325,24 +350,23 @@ def default_build_triple(verbose): cputype = cputype_mapper[cputype] elif cputype in {'xscale', 'arm'}: cputype = 'arm' - if ostype == 'linux-android': - ostype = 'linux-androideabi' - elif ostype == 'unknown-freebsd': - cputype = subprocess.check_output( - ['uname', '-p']).strip().decode(default_encoding) - ostype = 'unknown-freebsd' + if kernel == 'linux-android': + kernel = 'linux-androideabi' + elif kernel == 'unknown-freebsd': + cputype = processor + kernel = 'unknown-freebsd' elif cputype == 'armv6l': cputype = 'arm' - if ostype == 'linux-android': - ostype = 'linux-androideabi' + if kernel == 'linux-android': + kernel = 'linux-androideabi' else: - ostype += 'eabihf' + kernel += 'eabihf' elif cputype in {'armv7l', 'armv8l'}: cputype = 'armv7' - if ostype == 'linux-android': - ostype = 'linux-androideabi' + if kernel == 'linux-android': + kernel = 'linux-androideabi' else: - ostype += 'eabihf' + kernel += 'eabihf' elif cputype == 'mips': if sys.byteorder == 'big': cputype = 'mips' @@ -358,14 +382,14 @@ def default_build_triple(verbose): else: raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) # only the n64 ABI is supported, indicate it - ostype += 'abi64' + kernel += 'abi64' elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': pass else: err = "unknown cpu type: {}".format(cputype) sys.exit(err) - return "{}-{}".format(cputype, ostype) + return "{}-{}".format(cputype, kernel) @contextlib.contextmanager @@ -392,6 +416,48 @@ class Stage0Toolchain: return self.version + "-" + self.date +class DownloadInfo: + """A helper class that can be pickled into a parallel subprocess""" + + def __init__( + self, + base_download_url, + download_path, + bin_root, + tarball_path, + tarball_suffix, + checksums_sha256, + pattern, + verbose, + ): + self.base_download_url = base_download_url + self.download_path = download_path + self.bin_root = bin_root + self.tarball_path = tarball_path + self.tarball_suffix = tarball_suffix + self.checksums_sha256 = checksums_sha256 + self.pattern = pattern + self.verbose = verbose + +def download_component(download_info): + if not os.path.exists(download_info.tarball_path): + get( + download_info.base_download_url, + download_info.download_path, + download_info.tarball_path, + download_info.checksums_sha256, + verbose=download_info.verbose, + ) + +def unpack_component(download_info): + unpack( + download_info.tarball_path, + download_info.tarball_suffix, + download_info.bin_root, + match=download_info.pattern, + verbose=download_info.verbose, + ) + class RustBuild(object): """Provide all the methods required to build Rust""" def __init__(self): @@ -428,18 +494,71 @@ class RustBuild(object): (not os.path.exists(self.rustc()) or self.program_out_of_date(self.rustc_stamp(), key)): if os.path.exists(bin_root): + # HACK: On Windows, we can't delete rust-analyzer-proc-macro-server while it's + # running. Kill it. + if platform_is_win32(): + print("Killing rust-analyzer-proc-macro-srv before deleting stage0 toolchain") + regex = '{}\\\\(host|{})\\\\stage0\\\\libexec'.format( + os.path.basename(self.build_dir), + self.build + ) + script = ( + # NOTE: can't use `taskkill` or `Get-Process -Name` because they error if + # the server isn't running. + 'Get-Process | ' + + 'Where-Object {$_.Name -eq "rust-analyzer-proc-macro-srv"} |' + + 'Where-Object {{$_.Path -match "{}"}} |'.format(regex) + + 'Stop-Process' + ) + run_powershell([script]) shutil.rmtree(bin_root) + + key = self.stage0_compiler.date + cache_dst = os.path.join(self.build_dir, "cache") + rustc_cache = os.path.join(cache_dst, key) + if not os.path.exists(rustc_cache): + os.makedirs(rustc_cache) + tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz' - filename = "rust-std-{}-{}{}".format( - rustc_channel, self.build, tarball_suffix) - pattern = "rust-std-{}".format(self.build) - self._download_component_helper(filename, pattern, tarball_suffix) - filename = "rustc-{}-{}{}".format(rustc_channel, self.build, - tarball_suffix) - self._download_component_helper(filename, "rustc", tarball_suffix) - filename = "cargo-{}-{}{}".format(rustc_channel, self.build, - tarball_suffix) - self._download_component_helper(filename, "cargo", tarball_suffix) + + toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix) + + tarballs_to_download = [ + ("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)), + ("rustc-{}".format(toolchain_suffix), "rustc"), + ("cargo-{}".format(toolchain_suffix), "cargo"), + ] + + tarballs_download_info = [ + DownloadInfo( + base_download_url=self.download_url, + download_path="dist/{}/{}".format(self.stage0_compiler.date, filename), + bin_root=self.bin_root(), + tarball_path=os.path.join(rustc_cache, filename), + tarball_suffix=tarball_suffix, + checksums_sha256=self.checksums_sha256, + pattern=pattern, + verbose=self.verbose, + ) + for filename, pattern in tarballs_to_download + ] + + # Download the components serially to show the progress bars properly. + for download_info in tarballs_download_info: + download_component(download_info) + + # Unpack the tarballs in parallle. + # In Python 2.7, Pool cannot be used as a context manager. + pool_size = min(len(tarballs_download_info), get_cpus()) + if self.verbose: + print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs') + p = Pool(pool_size) + try: + p.map(unpack_component, tarballs_download_info) + finally: + p.close() + p.join() + if self.should_fix_bins_and_dylibs(): self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) @@ -455,13 +574,9 @@ class RustBuild(object): rust_stamp.write(key) def _download_component_helper( - self, filename, pattern, tarball_suffix, + self, filename, pattern, tarball_suffix, rustc_cache, ): key = self.stage0_compiler.date - cache_dst = os.path.join(self.build_dir, "cache") - rustc_cache = os.path.join(cache_dst, key) - if not os.path.exists(rustc_cache): - os.makedirs(rustc_cache) tarball = os.path.join(rustc_cache, filename) if not os.path.exists(tarball): @@ -516,7 +631,7 @@ class RustBuild(object): answer = self._should_fix_bins_and_dylibs = get_answer() if answer: - print("info: You seem to be using Nix.") + print("info: You seem to be using Nix.", file=sys.stderr) return answer def fix_bin_or_dylib(self, fname): @@ -529,7 +644,7 @@ class RustBuild(object): Please see https://nixos.org/patchelf.html for more information """ assert self._should_fix_bins_and_dylibs is True - print("attempting to patch", fname) + print("attempting to patch", fname, file=sys.stderr) # Only build `.nix-deps` once. nix_deps_dir = self.nix_deps_dir @@ -562,7 +677,7 @@ class RustBuild(object): "nix-build", "-E", nix_expr, "-o", nix_deps_dir, ]) except subprocess.CalledProcessError as reason: - print("warning: failed to call nix-build:", reason) + print("warning: failed to call nix-build:", reason, file=sys.stderr) return self.nix_deps_dir = nix_deps_dir @@ -575,14 +690,14 @@ class RustBuild(object): ] patchelf_args = ["--set-rpath", ":".join(rpath_entries)] if not fname.endswith(".so"): - # Finally, set the corret .interp for binaries + # Finally, set the correct .interp for binaries with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker: patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()] try: subprocess.check_output([patchelf] + patchelf_args + [fname]) except subprocess.CalledProcessError as reason: - print("warning: failed to call patchelf:", reason) + print("warning: failed to call patchelf:", reason, file=sys.stderr) return def rustc_stamp(self): @@ -722,11 +837,14 @@ class RustBuild(object): def build_bootstrap(self, color, verbose_count): """Build bootstrap""" - print("Building bootstrap") + env = os.environ.copy() + if "GITHUB_ACTIONS" in env: + print("::group::Building bootstrap") + else: + print("Building bootstrap", file=sys.stderr) build_dir = os.path.join(self.build_dir, "bootstrap") if self.clean and os.path.exists(build_dir): shutil.rmtree(build_dir) - env = os.environ.copy() # `CARGO_BUILD_TARGET` breaks bootstrap build. # See also: <https://github.com/rust-lang/rust/issues/70208>. if "CARGO_BUILD_TARGET" in env: @@ -798,6 +916,9 @@ class RustBuild(object): # Run this from the source directory so cargo finds .cargo/config run(args, env=env, verbose=self.verbose, cwd=self.rust_root) + if "GITHUB_ACTIONS" in env: + print("::endgroup::") + def build_triple(self): """Build triple as in LLVM @@ -814,25 +935,33 @@ class RustBuild(object): if 'SUDO_USER' in os.environ and not self.use_vendored_sources: if os.getuid() == 0: self.use_vendored_sources = True - print('info: looks like you\'re trying to run this command as root') - print(' and so in order to preserve your $HOME this will now') - print(' use vendored sources by default.') + print('info: looks like you\'re trying to run this command as root', + file=sys.stderr) + print(' and so in order to preserve your $HOME this will now', + file=sys.stderr) + print(' use vendored sources by default.', + file=sys.stderr) cargo_dir = os.path.join(self.rust_root, '.cargo') if self.use_vendored_sources: vendor_dir = os.path.join(self.rust_root, 'vendor') if not os.path.exists(vendor_dir): - sync_dirs = "--sync ./src/tools/rust-analyzer/Cargo.toml " \ + sync_dirs = "--sync ./src/tools/cargo/Cargo.toml " \ + "--sync ./src/tools/rust-analyzer/Cargo.toml " \ "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \ "--sync ./src/bootstrap/Cargo.toml " - print('error: vendoring required, but vendor directory does not exist.') + print('error: vendoring required, but vendor directory does not exist.', + file=sys.stderr) print(' Run `cargo vendor {}` to initialize the ' - 'vendor directory.'.format(sync_dirs)) - print('Alternatively, use the pre-vendored `rustc-src` dist component.') + 'vendor directory.'.format(sync_dirs), + file=sys.stderr) + print('Alternatively, use the pre-vendored `rustc-src` dist component.', + file=sys.stderr) raise Exception("{} not found".format(vendor_dir)) if not os.path.exists(cargo_dir): - print('error: vendoring required, but .cargo/config does not exist.') + print('error: vendoring required, but .cargo/config does not exist.', + file=sys.stderr) raise Exception("{} not found".format(cargo_dir)) else: if os.path.exists(cargo_dir): @@ -942,7 +1071,7 @@ def main(): print( "info: Downloading and building bootstrap before processing --help command.\n" " See src/bootstrap/README.md for help with common commands." - ) + , file=sys.stderr) exit_code = 0 success_word = "successfully" @@ -953,11 +1082,12 @@ def main(): exit_code = error.code else: exit_code = 1 - print(error) + print(error, file=sys.stderr) success_word = "unsuccessfully" if not help_triggered: - print("Build completed", success_word, "in", format_build_time(time() - start_time)) + print("Build completed", success_word, "in", format_build_time(time() - start_time), + file=sys.stderr) sys.exit(exit_code) |