Merging upstream version 1.72.1+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 110 insertions, 163 deletions

diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py
index c9e2961f6..66cba5842 100755
--- a/src/ci/docker/scripts/android-sdk-manager.py
+++ b/src/ci/docker/scripts/android-sdk-manager.py
@@ -2,6 +2,14 @@
 # Simpler reimplementation of Android's sdkmanager
 # Extra features of this implementation are pinning and mirroring
 
+import argparse
+import hashlib
+import os
+import subprocess
+import tempfile
+import urllib.request
+import xml.etree.ElementTree as ET
+
 # These URLs are the Google repositories containing the list of available
 # packages and their versions. The list has been generated by listing the URLs
 # fetched while executing `tools/bin/sdkmanager --list`
@@ -27,15 +35,6 @@ MIRROR_BUCKET = "rust-lang-ci-mirrors"
 MIRROR_BUCKET_REGION = "us-west-1"
 MIRROR_BASE_DIR = "rustc/android/"
 
-import argparse
-import hashlib
-import os
-import subprocess
-import sys
-import tempfile
-import urllib.request
-import xml.etree.ElementTree as ET
-
 class Package:
     def __init__(self, path, url, sha1, deps=None):
         if deps is None:
diff --git a/src/ci/docker/scripts/crosstool-ng-git.sh b/src/ci/docker/scripts/crosstool-ng-git.sh
index 449cc476f..b8d399153 100644
--- a/src/ci/docker/scripts/crosstool-ng-git.sh
+++ b/src/ci/docker/scripts/crosstool-ng-git.sh
@@ -2,7 +2,7 @@
 set -ex
 
 URL=https://github.com/crosstool-ng/crosstool-ng
-REV=943364711a650d9b9e84c1b42c91cc0265b6ab5c
+REV=227d99d7f3115f3a078595a580d2b307dcd23e93
 
 mkdir crosstool-ng
 cd crosstool-ng
diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py
index ecef56f56..af01f9ccb 100755
--- a/src/ci/docker/scripts/fuchsia-test-runner.py
+++ b/src/ci/docker/scripts/fuchsia-test-runner.py
@@ -25,13 +25,9 @@ from typing import ClassVar, List, Optional
 
 @dataclass
 class TestEnvironment:
-    rust_dir: str
+    rust_build_dir: str
     sdk_dir: str
     target: str
-    package_server_pid: Optional[int] = None
-    emu_addr: Optional[str] = None
-    libstd_name: Optional[str] = None
-    libtest_name: Optional[str] = None
     verbose: bool = False
 
     @staticmethod
@@ -57,7 +53,7 @@ class TestEnvironment:
     @classmethod
     def from_args(cls, args):
         return cls(
-            os.path.abspath(args.rust),
+            os.path.abspath(args.rust_build),
             os.path.abspath(args.sdk),
             args.target,
             verbose=args.verbose,
@@ -68,13 +64,9 @@ class TestEnvironment:
         with open(cls.env_file_path(), encoding="utf-8") as f:
             test_env = json.loads(f.read())
             return cls(
-                test_env["rust_dir"],
+                test_env["rust_build_dir"],
                 test_env["sdk_dir"],
                 test_env["target"],
-                libstd_name=test_env["libstd_name"],
-                libtest_name=test_env["libtest_name"],
-                emu_addr=test_env["emu_addr"],
-                package_server_pid=test_env["package_server_pid"],
                 verbose=test_env["verbose"],
             )
 
@@ -82,18 +74,6 @@ class TestEnvironment:
         with open(self.env_file_path(), "w", encoding="utf-8") as f:
             f.write(json.dumps(self.__dict__))
 
-    def ssh_dir(self):
-        return os.path.join(self.tmp_dir(), "ssh")
-
-    def ssh_keyfile_path(self):
-        return os.path.join(self.ssh_dir(), "fuchsia_ed25519")
-
-    def ssh_authfile_path(self):
-        return os.path.join(self.ssh_dir(), "fuchsia_authorized_keys")
-
-    def vdl_output_path(self):
-        return os.path.join(self.tmp_dir(), "vdl_output")
-
     def package_server_log_path(self):
         return os.path.join(self.tmp_dir(), "package_server_log")
 
@@ -113,7 +93,9 @@ class TestEnvironment:
 
     def libs_dir(self):
         return os.path.join(
-            self.rust_dir,
+            self.rust_build_dir,
+            "host",
+            "stage2",
             "lib",
         )
 
@@ -171,7 +153,6 @@ class TestEnvironment:
     def home_dir(self):
         return os.path.join(self.tmp_dir(), "user-home")
 
-
     def start_ffx_isolation(self):
         # Most of this is translated directly from ffx's isolate library
         os.mkdir(self.ffx_isolate_dir())
@@ -213,21 +194,19 @@ class TestEnvironment:
         # Set configs
         configs = {
             "log.enabled": "true",
-            "ssh.pub": self.ssh_authfile_path(),
-            "ssh.priv": self.ssh_keyfile_path(),
             "test.is_isolated": "true",
             "test.experimental_structured_output": "true",
         }
         for key, value in configs.items():
             subprocess.check_call(
                 [
-                    self.tool_path("ffx"),
+                    ffx_path,
                     "config",
                     "set",
                     key,
                     value,
                 ],
-                env=self.ffx_cmd_env(),
+                env=ffx_env,
                 stdout=self.subprocess_output(),
                 stderr=self.subprocess_output(),
             )
@@ -249,6 +228,7 @@ class TestEnvironment:
                 self.tool_path("ffx"),
                 "daemon",
                 "stop",
+                "-w",
             ],
             env=self.ffx_cmd_env(),
             stdout=self.subprocess_output(),
@@ -276,87 +256,62 @@ class TestEnvironment:
         elif len(os.listdir(self.tmp_dir())) != 0:
             raise Exception(f"Temp directory is not clean (in {self.tmp_dir()})")
 
-        os.mkdir(self.ssh_dir())
         os.mkdir(self.output_dir())
 
-        # Find libstd and libtest
-        libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so"))
-        libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so"))
-
-        if not libstd_paths:
-            raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})")
-
-        if not libtest_paths:
-            raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})")
+        ffx_path = self.tool_path("ffx")
+        ffx_env = self.ffx_cmd_env()
 
-        self.libstd_name = os.path.basename(libstd_paths[0])
-        self.libtest_name = os.path.basename(libtest_paths[0])
+        # Start ffx isolation
+        self.log_info("Starting ffx isolation...")
+        self.start_ffx_isolation()
 
-        # Generate SSH keys for the emulator to use
-        self.log_info("Generating SSH keys...")
+        # Stop any running emulators (there shouldn't be any)
         subprocess.check_call(
             [
-                "ssh-keygen",
-                "-N",
-                "",
-                "-t",
-                "ed25519",
-                "-f",
-                self.ssh_keyfile_path(),
-                "-C",
-                "Generated by fuchsia-test-runner.py",
+                ffx_path,
+                "emu",
+                "stop",
+                "--all",
             ],
+            env=ffx_env,
             stdout=self.subprocess_output(),
             stderr=self.subprocess_output(),
         )
-        authfile_contents = subprocess.check_output(
+
+        # Start emulator
+        self.log_info("Starting emulator...")
+        product_bundle = "terminal.qemu-" + self.triple_to_arch(self.target)
+        subprocess.check_call(
             [
-                "ssh-keygen",
-                "-y",
-                "-f",
-                self.ssh_keyfile_path(),
+                ffx_path,
+                "product-bundle",
+                "get",
+                product_bundle,
             ],
+            env=ffx_env,
+            stdout=self.subprocess_output(),
             stderr=self.subprocess_output(),
         )
-        with open(self.ssh_authfile_path(), "wb") as authfile:
-            authfile.write(authfile_contents)
-
-        # Start ffx isolation
-        self.log_info("Starting ffx isolation...")
-        self.start_ffx_isolation()
-
-        # Start emulator (this will generate the vdl output)
-        self.log_info("Starting emulator...")
+        # FIXME: condition --accel hyper on target arch matching host arch
         subprocess.check_call(
             [
-                self.tool_path("fvdl"),
-                "--sdk",
+                ffx_path,
+                "emu",
                 "start",
-                "--tuntap",
+                product_bundle,
                 "--headless",
-                "--nointeractive",
-                "--ssh",
-                self.ssh_dir(),
-                "--vdl-output",
-                self.vdl_output_path(),
-                "--emulator-log",
+                "--log",
                 self.emulator_log_path(),
-                "--image-name",
-                "qemu-" + self.triple_to_arch(self.target),
+                "--net",
+                "tap",
+                "--accel",
+                "hyper",
             ],
+            env=ffx_env,
             stdout=self.subprocess_output(),
             stderr=self.subprocess_output(),
         )
 
-        # Parse vdl output for relevant information
-        with open(self.vdl_output_path(), encoding="utf-8") as f:
-            vdl_content = f.read()
-            matches = re.search(
-                r'network_address:\s+"\[([0-9a-f]{1,4}:(:[0-9a-f]{1,4}){4}%qemu)\]"',
-                vdl_content,
-            )
-            self.emu_addr = matches.group(1)
-
         # Create new package repo
         self.log_info("Creating package repo...")
         subprocess.check_call(
@@ -370,61 +325,46 @@ class TestEnvironment:
             stderr=self.subprocess_output(),
         )
 
-        # Start package server
-        self.log_info("Starting package server...")
-        with open(
-            self.package_server_log_path(), "w", encoding="utf-8"
-        ) as package_server_log:
-            # We want this to be a long-running process that persists after the script finishes
-            # pylint: disable=consider-using-with
-            self.package_server_pid = subprocess.Popen(
-                [
-                    self.tool_path("pm"),
-                    "serve",
-                    "-vt",
-                    "-repo",
-                    self.repo_dir(),
-                    "-l",
-                    ":8084",
-                ],
-                stdout=package_server_log,
-                stderr=package_server_log,
-            ).pid
-
-        # Register package server with emulator
-        self.log_info("Registering package server...")
-        ssh_client = subprocess.check_output(
+        # Add repo
+        subprocess.check_call(
             [
-                "ssh",
-                "-i",
-                self.ssh_keyfile_path(),
-                "-o",
-                "StrictHostKeyChecking=accept-new",
-                self.emu_addr,
-                "-f",
-                "echo $SSH_CLIENT",
+                ffx_path,
+                "repository",
+                "add-from-pm",
+                self.repo_dir(),
+                "--repository",
+                self.TEST_REPO_NAME,
             ],
-            text=True,
+            env=ffx_env,
+            stdout=self.subprocess_output(),
+            stderr=self.subprocess_output(),
         )
-        repo_addr = ssh_client.split()[0].replace("%", "%25")
-        repo_url = f"http://[{repo_addr}]:8084/config.json"
+
+        # Start repository server
+        subprocess.check_call(
+            [ffx_path, "repository", "server", "start", "--address", "[::]:0"],
+            env=ffx_env,
+            stdout=self.subprocess_output(),
+            stderr=self.subprocess_output(),
+        )
+
+        # Register with newly-started emulator
         subprocess.check_call(
             [
-                "ssh",
-                "-i",
-                self.ssh_keyfile_path(),
-                "-o",
-                "StrictHostKeyChecking=accept-new",
-                self.emu_addr,
-                "-f",
-                f"pkgctl repo add url -f 1 -n {self.TEST_REPO_NAME} {repo_url}",
+                ffx_path,
+                "target",
+                "repository",
+                "register",
+                "--repository",
+                self.TEST_REPO_NAME,
             ],
+            env=ffx_env,
             stdout=self.subprocess_output(),
             stderr=self.subprocess_output(),
         )
 
         # Create lockfiles
-        open(self.pm_lockfile_path(), 'a').close()
+        open(self.pm_lockfile_path(), "a").close()
 
         # Write to file
         self.write_to_file()
@@ -458,6 +398,7 @@ class TestEnvironment:
         ],
         use: [
             {{ storage: "data", path: "/data" }},
+            {{ storage: "tmp", path: "/tmp" }},
             {{ protocol: [ "fuchsia.process.Launcher" ] }},
             {{ protocol: [ "fuchsia.posix.socket.Provider" ] }}
         ],
@@ -471,8 +412,8 @@ class TestEnvironment:
     meta/package={package_dir}/meta/package
     meta/{package_name}.cm={package_dir}/meta/{package_name}.cm
     bin/{exe_name}={bin_path}
-    lib/{libstd_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libstd_name}
-    lib/{libtest_name}={rust_dir}/lib/rustlib/{rustlib_dir}/lib/{libtest_name}
+    lib/{libstd_name}={libstd_path}
+    lib/{libtest_name}={libtest_path}
     lib/ld.so.1={sdk_dir}/arch/{target_arch}/sysroot/dist/lib/ld.so.1
     lib/libfdio.so={sdk_dir}/arch/{target_arch}/dist/libfdio.so
     """
@@ -502,6 +443,16 @@ class TestEnvironment:
 
         bin_path = os.path.abspath(args.bin_path)
 
+        # Find libstd and libtest
+        libstd_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libstd-*.so"))
+        libtest_paths = glob.glob(os.path.join(self.rustlibs_dir(), "libtest-*.so"))
+
+        if not libstd_paths:
+            raise Exception(f"Failed to locate libstd (in {self.rustlibs_dir()})")
+
+        if not libtest_paths:
+            raise Exception(f"Failed to locate libtest (in {self.rustlibs_dir()})")
+
         # Build a unique, deterministic name for the test using the name of the
         # binary and the last 6 hex digits of the hash of the full path
         def path_checksum(path):
@@ -568,8 +519,11 @@ class TestEnvironment:
                         env_vars += f'\n            "{var_name}={var_value}",'
 
                 # Default to no backtrace for test suite
-                if os.getenv("RUST_BACKTRACE") == None:
-                    env_vars += f'\n            "RUST_BACKTRACE=0",'
+                if os.getenv("RUST_BACKTRACE") is None:
+                    env_vars += '\n            "RUST_BACKTRACE=0",'
+
+                # Use /tmp as the test temporary directory
+                env_vars += f'\n            "RUST_TEST_TMPDIR=/tmp",'
 
                 cml.write(
                     self.CML_TEMPLATE.format(env_vars=env_vars, exe_name=exe_name)
@@ -601,11 +555,12 @@ class TestEnvironment:
                         exe_name=exe_name,
                         package_dir=package_dir,
                         package_name=package_name,
-                        rust_dir=self.rust_dir,
-                        rustlib_dir=self.target,
+                        target=self.target,
                         sdk_dir=self.sdk_dir,
-                        libstd_name=self.libstd_name,
-                        libtest_name=self.libtest_name,
+                        libstd_name=os.path.basename(libstd_paths[0]),
+                        libtest_name=os.path.basename(libtest_paths[0]),
+                        libstd_path=libstd_paths[0],
+                        libtest_path=libtest_paths[0],
                         target_arch=self.triple_to_arch(self.target),
                     )
                 )
@@ -642,7 +597,7 @@ class TestEnvironment:
             log("Publishing package to repo...")
 
             # Publish package to repo
-            with open(self.pm_lockfile_path(), 'w') as pm_lockfile:
+            with open(self.pm_lockfile_path(), "w") as pm_lockfile:
                 fcntl.lockf(pm_lockfile.fileno(), fcntl.LOCK_EX)
                 subprocess.check_call(
                     [
@@ -776,20 +731,15 @@ class TestEnvironment:
         else:
             self.log_debug("No ffx daemon log found")
 
-        # Stop package server
-        self.log_info("Stopping package server...")
-        os.kill(self.package_server_pid, signal.SIGTERM)
-
         # Shut down the emulator
         self.log_info("Stopping emulator...")
         subprocess.check_call(
             [
-                self.tool_path("fvdl"),
-                "--sdk",
-                "kill",
-                "--launched-proto",
-                self.vdl_output_path(),
+                self.tool_path("ffx"),
+                "emu",
+                "stop",
             ],
+            env=self.ffx_cmd_env(),
             stdout=self.subprocess_output(),
             stderr=self.subprocess_output(),
         )
@@ -966,8 +916,8 @@ def main():
         "start", help="initializes the testing environment"
     )
     start_parser.add_argument(
-        "--rust",
-        help="the directory of the installed Rust compiler for Fuchsia",
+        "--rust-build",
+        help="the current compiler build directory (`$RUST_SRC/build` by default)",
         required=True,
     )
     start_parser.add_argument(
@@ -1045,9 +995,7 @@ def main():
     )
     debug_parser.set_defaults(func=debug)
 
-    syslog_parser = subparsers.add_parser(
-        "syslog", help="prints the device syslog"
-    )
+    syslog_parser = subparsers.add_parser("syslog", help="prints the device syslog")
     syslog_parser.set_defaults(func=syslog)
 
     args = parser.parse_args()