From dc0db358abe19481e475e10c32149b53370f1a1c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 30 May 2024 05:57:31 +0200 Subject: Merging upstream version 1.72.1+dfsg1. Signed-off-by: Daniel Baumann --- src/ci/docker/scripts/android-sdk-manager.py | 17 +- src/ci/docker/scripts/crosstool-ng-git.sh | 2 +- src/ci/docker/scripts/fuchsia-test-runner.py | 254 +++++++++++---------------- 3 files changed, 110 insertions(+), 163 deletions(-) (limited to 'src/ci/docker/scripts') diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py index c9e2961f6..66cba5842 100755 --- a/src/ci/docker/scripts/android-sdk-manager.py +++ b/src/ci/docker/scripts/android-sdk-manager.py @@ -2,6 +2,14 @@ # Simpler reimplementation of Android's sdkmanager # Extra features of this implementation are pinning and mirroring +import argparse +import hashlib +import os +import subprocess +import tempfile +import urllib.request +import xml.etree.ElementTree as ET + # These URLs are the Google repositories containing the list of available # packages and their versions. The list has been generated by listing the URLs # fetched while executing `tools/bin/sdkmanager --list` @@ -27,15 +35,6 @@ MIRROR_BUCKET = "rust-lang-ci-mirrors" MIRROR_BUCKET_REGION = "us-west-1" MIRROR_BASE_DIR = "rustc/android/" -import argparse -import hashlib -import os -import subprocess -import sys -import tempfile -import urllib.request -import xml.etree.ElementTree as ET - class Package: def __init__(self, path, url, sha1, deps=None): if deps is None: diff --git a/src/ci/docker/scripts/crosstool-ng-git.sh b/src/ci/docker/scripts/crosstool-ng-git.sh index 449cc476f..b8d399153 100644 --- a/src/ci/docker/scripts/crosstool-ng-git.sh +++ b/src/ci/docker/scripts/crosstool-ng-git.sh @@ -2,7 +2,7 @@ set -ex URL=https://github.com/crosstool-ng/crosstool-ng -REV=943364711a650d9b9e84c1b42c91cc0265b6ab5c +REV=227d99d7f3115f3a078595a580d2b307dcd23e93 mkdir crosstool-ng cd crosstool-ng diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py index ecef56f56..af01f9ccb 100755 --- a/src/ci/docker/scripts/fuchsia-test-runner.py +++ b/src/ci/docker/scripts/fuchsia-test-runner.py @@ -25,13 +25,9 @@ from typing import ClassVar, List, Optional @dataclass class TestEnvironment: - rust_dir: str + rust_build_dir: str sdk_dir: str target: str - package_server_pid: Optional[int] = None - emu_addr: Optional[str] = None - libstd_name: Optional[str] = None - libtest_name: Optional[str] = None verbose: bool = False @staticmethod @@ -57,7 +53,7 @@ class TestEnvironment: @classmethod def from_args(cls, args): return cls( - os.path.abspath(args.rust), + os.path.abspath(args.rust_build), os.path.abspath(args.sdk), args.target, verbose=args.verbose, @@ -68,13 +64,9 @@ class TestEnvironment: with open(cls.env_file_path(), encoding="utf-8") as f: test_env = json.loads(f.read()) return cls( - test_env["rust_dir"], + test_env["rust_build_dir"], test_env["sdk_dir"], test_env["target"], - libstd_name=test_env["libstd_name"], - libtest_name=test_env["libtest_name"], - emu_addr=test_env["emu_addr"], - package_server_pid=test_env["package_server_pid"], verbose=test_env["verbose"], ) @@ -82,18 +74,6 @@ class TestEnvironment: with open(self.env_file_path(), "w", encoding="utf-8") as f: f.write(json.dumps(self.__dict__)) - def ssh_dir(self): - return os.path.join(self.tmp_dir(), "ssh") - - def ssh_keyfile_path(self): - return os.path.join(self.ssh_dir(), "fuchsia_ed25519") - - def ssh_authfile_path(self): - return os.path.join(self.ssh_dir(), "fuchsia_authorized_keys") - - def vdl_output_path(self): - return os.path.join(self.tmp_dir(), "vdl_output") - def package_server_log_path(self): return os.path.join(self.tmp_dir(), "package_server_log") @@ -113,7 +93,9 @@ class TestEnvironment: def libs_dir(self): return os.path.join( - self.rust_dir, + self.rust_build_dir, + "host", + "stage2", "lib", ) @@ -171,7 +153,6 @@ class TestEnvironment: def home_dir(self): return os.path.join(self.tmp_dir(), "user-home") - def start_ffx_isolation(self): # Most of this is translated directly from ffx's isolate library os.mkdir(self.ffx_isolate_dir()) @@ -213,21 +194,19 @@ class TestEnvironment: # Set configs configs = { "log.enabled": "true", - "ssh.pub": self.ssh_authfile_path(), - "ssh.priv": self.ssh_keyfile_path(), "test.is_isolated": "true", "test.experimental_structured_output": "true", } for key, value in configs.items(): subprocess.check_call( [ - self.tool_path("ffx"), + ffx_path, "config", "set", key, value, ], - env=self.ffx_cmd_env(), + env=ffx_env, stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) @@ -249,6 +228,7 @@ class TestEnvironment: self.tool_path("ffx"), "daemon", "stop", + "-w", ], env=self.ffx_cmd_env(), stdout=self.subprocess_output(), @@ -276,87 +256,62 @@ class TestEnvironment: elif len(os.listdir(self.tmp_dir())) != 0: raise Exception(f"Temp directory is not clean (in {self.tmp_dir()})") - os.mkdir(self.ssh_dir()) os.mkdir(self.output_dir()) - # Find libstd and libtest - libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so")) - libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so")) - - if not libstd_paths: - raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})") - - if not libtest_paths: - raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})") + ffx_path = self.tool_path("ffx") + ffx_env = self.ffx_cmd_env() - self.libstd_name = os.path.basename(libstd_paths[0]) - self.libtest_name = os.path.basename(libtest_paths[0]) + # Start ffx isolation + self.log_info("Starting ffx isolation...") + self.start_ffx_isolation() - # Generate SSH keys for the emulator to use - self.log_info("Generating SSH keys...") + # Stop any running emulators (there shouldn't be any) subprocess.check_call( [ - "ssh-keygen", - "-N", - "", - "-t", - "ed25519", - "-f", - self.ssh_keyfile_path(), - "-C", - "Generated by fuchsia-test-runner.py", + ffx_path, + "emu", + "stop", + "--all", ], + env=ffx_env, stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) - authfile_contents = subprocess.check_output( + + # Start emulator + self.log_info("Starting emulator...") + product_bundle = "terminal.qemu-" + self.triple_to_arch(self.target) + subprocess.check_call( [ - "ssh-keygen", - "-y", - "-f", - self.ssh_keyfile_path(), + ffx_path, + "product-bundle", + "get", + product_bundle, ], + env=ffx_env, + stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) - with open(self.ssh_authfile_path(), "wb") as authfile: - authfile.write(authfile_contents) - - # Start ffx isolation - self.log_info("Starting ffx isolation...") - self.start_ffx_isolation() - - # Start emulator (this will generate the vdl output) - self.log_info("Starting emulator...") + # FIXME: condition --accel hyper on target arch matching host arch subprocess.check_call( [ - self.tool_path("fvdl"), - "--sdk", + ffx_path, + "emu", "start", - "--tuntap", + product_bundle, "--headless", - "--nointeractive", - "--ssh", - self.ssh_dir(), - "--vdl-output", - self.vdl_output_path(), - "--emulator-log", + "--log", self.emulator_log_path(), - "--image-name", - "qemu-" + self.triple_to_arch(self.target), + "--net", + "tap", + "--accel", + "hyper", ], + env=ffx_env, stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) - # Parse vdl output for relevant information - with open(self.vdl_output_path(), encoding="utf-8") as f: - vdl_content = f.read() - matches = re.search( - r'network_address:\s+"\[([0-9a-f]{1,4}:(:[0-9a-f]{1,4}){4}%qemu)\]"', - vdl_content, - ) - self.emu_addr = matches.group(1) - # Create new package repo self.log_info("Creating package repo...") subprocess.check_call( @@ -370,61 +325,46 @@ class TestEnvironment: stderr=self.subprocess_output(), ) - # Start package server - self.log_info("Starting package server...") - with open( - self.package_server_log_path(), "w", encoding="utf-8" - ) as package_server_log: - # We want this to be a long-running process that persists after the script finishes - # pylint: disable=consider-using-with - self.package_server_pid = subprocess.Popen( - [ - self.tool_path("pm"), - "serve", - "-vt", - "-repo", - self.repo_dir(), - "-l", - ":8084", - ], - stdout=package_server_log, - stderr=package_server_log, - ).pid - - # Register package server with emulator - self.log_info("Registering package server...") - ssh_client = subprocess.check_output( + # Add repo + subprocess.check_call( [ - "ssh", - "-i", - self.ssh_keyfile_path(), - "-o", - "StrictHostKeyChecking=accept-new", - self.emu_addr, - "-f", - "echo $SSH_CLIENT", + ffx_path, + "repository", + "add-from-pm", + self.repo_dir(), + "--repository", + self.TEST_REPO_NAME, ], - text=True, + env=ffx_env, + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), ) - repo_addr = ssh_client.split()[0].replace("%", "%25") - repo_url = f"http://[{repo_addr}]:8084/config.json" + + # Start repository server + subprocess.check_call( + [ffx_path, "repository", "server", "start", "--address", "[::]:0"], + env=ffx_env, + stdout=self.subprocess_output(), + stderr=self.subprocess_output(), + ) + + # Register with newly-started emulator subprocess.check_call( [ - "ssh", - "-i", - self.ssh_keyfile_path(), - "-o", - "StrictHostKeyChecking=accept-new", - self.emu_addr, - "-f", - f"pkgctl repo add url -f 1 -n {self.TEST_REPO_NAME} {repo_url}", + ffx_path, + "target", + "repository", + "register", + "--repository", + self.TEST_REPO_NAME, ], + env=ffx_env, stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) # Create lockfiles - open(self.pm_lockfile_path(), 'a').close() + open(self.pm_lockfile_path(), "a").close() # Write to file self.write_to_file() @@ -458,6 +398,7 @@ class TestEnvironment: ], use: [ {{ storage: "data", path: "/data" }}, + {{ storage: "tmp", path: "/tmp" }}, {{ protocol: [ "fuchsia.process.Launcher" ] }}, {{ protocol: [ "fuchsia.posix.socket.Provider" ] }} ], @@ -471,8 +412,8 @@ class TestEnvironment: meta/package={package_dir}/meta/package meta/{package_name}.cm={package_dir}/meta/{package_name}.cm bin/{exe_name}={bin_path} - lib/{libstd_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libstd_name} - lib/{libtest_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libtest_name} + lib/{libstd_name}={libstd_path} + lib/{libtest_name}={libtest_path} lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/dist/lib/ld.so.1 lib/libfdio.so={sdk_dir}/arch/{target_arch}/dist/libfdio.so """ @@ -502,6 +443,16 @@ class TestEnvironment: bin_path = os.path.abspath(args.bin_path) + # Find libstd and libtest + libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so")) + libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so")) + + if not libstd_paths: + raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})") + + if not libtest_paths: + raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})") + # Build a unique, deterministic name for the test using the name of the # binary and the last 6 hex digits of the hash of the full path def path_checksum(path): @@ -568,8 +519,11 @@ class TestEnvironment: env_vars += f'\n "{var_name}={var_value}",' # Default to no backtrace for test suite - if os.getenv("RUST_BACKTRACE") == None: - env_vars += f'\n "RUST_BACKTRACE=0",' + if os.getenv("RUST_BACKTRACE") is None: + env_vars += '\n "RUST_BACKTRACE=0",' + + # Use /tmp as the test temporary directory + env_vars += f'\n "RUST_TEST_TMPDIR=/tmp",' cml.write( self.CML_TEMPLATE.format(env_vars=env_vars, exe_name=exe_name) @@ -601,11 +555,12 @@ class TestEnvironment: exe_name=exe_name, package_dir=package_dir, package_name=package_name, - rust_dir=self.rust_dir, - rustlib_dir=self.target, + target=self.target, sdk_dir=self.sdk_dir, - libstd_name=self.libstd_name, - libtest_name=self.libtest_name, + libstd_name=os.path.basename(libstd_paths[0]), + libtest_name=os.path.basename(libtest_paths[0]), + libstd_path=libstd_paths[0], + libtest_path=libtest_paths[0], target_arch=self.triple_to_arch(self.target), ) ) @@ -642,7 +597,7 @@ class TestEnvironment: log("Publishing package to repo...") # Publish package to repo - with open(self.pm_lockfile_path(), 'w') as pm_lockfile: + with open(self.pm_lockfile_path(), "w") as pm_lockfile: fcntl.lockf(pm_lockfile.fileno(), fcntl.LOCK_EX) subprocess.check_call( [ @@ -776,20 +731,15 @@ class TestEnvironment: else: self.log_debug("No ffx daemon log found") - # Stop package server - self.log_info("Stopping package server...") - os.kill(self.package_server_pid, signal.SIGTERM) - # Shut down the emulator self.log_info("Stopping emulator...") subprocess.check_call( [ - self.tool_path("fvdl"), - "--sdk", - "kill", - "--launched-proto", - self.vdl_output_path(), + self.tool_path("ffx"), + "emu", + "stop", ], + env=self.ffx_cmd_env(), stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) @@ -966,8 +916,8 @@ def main(): "start", help="initializes the testing environment" ) start_parser.add_argument( - "--rust", - help="the directory of the installed Rust compiler for Fuchsia", + "--rust-build", + help="the current compiler build directory (`$RUST_SRC/build` by default)", required=True, ) start_parser.add_argument( @@ -1045,9 +995,7 @@ def main(): ) debug_parser.set_defaults(func=debug) - syslog_parser = subparsers.add_parser( - "syslog", help="prints the device syslog" - ) + syslog_parser = subparsers.add_parser("syslog", help="prints the device syslog") syslog_parser.set_defaults(func=syslog) args = parser.parse_args() -- cgit v1.2.3