diff options
Diffstat (limited to 'tests/topotests/munet')
24 files changed, 12402 insertions, 0 deletions
diff --git a/tests/topotests/munet/__init__.py b/tests/topotests/munet/__init__.py new file mode 100644 index 0000000..e1f18e5 --- /dev/null +++ b/tests/topotests/munet/__init__.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# September 30 2021, Christian Hopps <chopps@labn.net> +# +# Copyright 2021, LabN Consulting, L.L.C. +# +"""A module to import various objects to root namespace.""" +from .base import BaseMunet +from .base import Bridge +from .base import Commander +from .base import LinuxNamespace +from .base import SharedNamespace +from .base import cmd_error +from .base import comm_error +from .base import get_exec_path +from .base import proc_error +from .native import L3Bridge +from .native import L3NamespaceNode +from .native import Munet +from .native import to_thread + + +__all__ = [ + "BaseMunet", + "Bridge", + "Commander", + "L3Bridge", + "L3NamespaceNode", + "LinuxNamespace", + "Munet", + "SharedNamespace", + "cmd_error", + "comm_error", + "get_exec_path", + "proc_error", + "to_thread", +] diff --git a/tests/topotests/munet/__main__.py b/tests/topotests/munet/__main__.py new file mode 100644 index 0000000..4419ab9 --- /dev/null +++ b/tests/topotests/munet/__main__.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# September 2 2021, Christian Hopps <chopps@labn.net> +# +# Copyright 2021, LabN Consulting, L.L.C. +# +"""The main function for standalone operation.""" +import argparse +import asyncio +import logging +import logging.config +import os +import subprocess +import sys + +from . import cli +from . import parser +from .base import get_event_loop +from .cleanup import cleanup_previous +from .compat import PytestConfig + + +logger = None + + +async def forever(): + while True: + await asyncio.sleep(3600) + + +async def run_and_wait(args, unet): + tasks = [] + + if not args.topology_only: + # add the cmd.wait()s returned from unet.run() + tasks += await unet.run() + + if sys.stdin.isatty() and not args.no_cli: + # Run an interactive CLI + task = cli.async_cli(unet) + else: + if args.no_wait: + logger.info("Waiting for all node cmd to complete") + task = asyncio.gather(*tasks, return_exceptions=True) + else: + logger.info("Waiting on signal to exit") + task = asyncio.create_task(forever()) + task = asyncio.gather(task, *tasks, return_exceptions=True) + + try: + await task + finally: + # Basically we are canceling tasks from unet.run() which are just async calls to + # node.cmd_p.wait() so we've stopped waiting for them to complete, but not + # actually canceld/killed the cmd_p process. + for task in tasks: + task.cancel() + + +async def async_main(args, config): + status = 3 + + # Setup the namespaces and network addressing. + + unet = await parser.async_build_topology( + config, rundir=args.rundir, args=args, pytestconfig=PytestConfig(args) + ) + logger.info("Topology up: rundir: %s", unet.rundir) + + try: + status = await run_and_wait(args, unet) + except KeyboardInterrupt: + logger.info("Exiting, received KeyboardInterrupt in async_main") + except asyncio.CancelledError as ex: + logger.info("task canceled error: %s cleaning up", ex) + except Exception as error: + logger.info("Exiting, unexpected exception %s", error, exc_info=True) + else: + logger.info("Exiting normally") + + logger.debug("main: async deleting") + try: + await unet.async_delete() + except KeyboardInterrupt: + status = 2 + logger.warning("Received KeyboardInterrupt while cleaning up.") + except Exception as error: + status = 2 + logger.info("Deleting, unexpected exception %s", error, exc_info=True) + return status + + +def main(*args): + ap = argparse.ArgumentParser(args) + cap = ap.add_argument_group(title="Config", description="config related options") + + cap.add_argument("-c", "--config", help="config file (yaml, toml, json, ...)") + cap.add_argument( + "-d", "--rundir", help="runtime directory for tempfiles, logs, etc" + ) + cap.add_argument( + "--kinds-config", + help="kinds config file, overrides default search (yaml, toml, json, ...)", + ) + cap.add_argument( + "--project-root", help="directory to stop searching for kinds config at" + ) + rap = ap.add_argument_group(title="Runtime", description="runtime related options") + rap.add_argument( + "-C", + "--cleanup", + action="store_true", + help="Remove the entire rundir (not just node subdirs) prior to running.", + ) + rap.add_argument( + "--gdb", metavar="NODE-LIST", help="comma-sep list of hosts to run gdb on" + ) + rap.add_argument( + "--gdb-breakpoints", + metavar="BREAKPOINT-LIST", + help="comma-sep list of breakpoints to set", + ) + rap.add_argument( + "--host", + action="store_true", + help="no isolation for top namespace, bridges exposed to default namespace", + ) + rap.add_argument( + "--pcap", + metavar="TARGET-LIST", + help="comma-sep list of capture targets (NETWORK or NODE:IFNAME)", + ) + rap.add_argument( + "--shell", metavar="NODE-LIST", help="comma-sep list of nodes to open shells on" + ) + rap.add_argument( + "--stderr", + metavar="NODE-LIST", + help="comma-sep list of nodes to open windows viewing stderr", + ) + rap.add_argument( + "--stdout", + metavar="NODE-LIST", + help="comma-sep list of nodes to open windows viewing stdout", + ) + rap.add_argument( + "--topology-only", + action="store_true", + help="Do not run any node commands", + ) + rap.add_argument("--unshare-inline", action="store_true", help=argparse.SUPPRESS) + rap.add_argument( + "--validate-only", + action="store_true", + help="Validate the config against the schema definition", + ) + rap.add_argument("-v", "--verbose", action="store_true", help="be verbose") + rap.add_argument( + "-V", "--version", action="store_true", help="print the verison number and exit" + ) + eap = ap.add_argument_group(title="Uncommon", description="uncommonly used options") + eap.add_argument("--log-config", help="logging config file (yaml, toml, json, ...)") + eap.add_argument( + "--no-kill", + action="store_true", + help="Do not kill previous running processes", + ) + eap.add_argument( + "--no-cli", action="store_true", help="Do not run the interactive CLI" + ) + eap.add_argument("--no-wait", action="store_true", help="Exit after commands") + + args = ap.parse_args() + + if args.version: + from importlib import metadata # pylint: disable=C0415 + + print(metadata.version("munet")) + sys.exit(0) + + rundir = args.rundir if args.rundir else "/tmp/munet" + args.rundir = rundir + + if args.cleanup: + if os.path.exists(rundir): + if not os.path.exists(f"{rundir}/config.json"): + logging.critical( + 'unsafe: won\'t clean up rundir "%s" as ' + "previous config.json not present", + rundir, + ) + sys.exit(1) + else: + subprocess.run(["/usr/bin/rm", "-rf", rundir], check=True) + + subprocess.run(f"mkdir -p {rundir} && chmod 755 {rundir}", check=True, shell=True) + os.environ["MUNET_RUNDIR"] = rundir + + parser.setup_logging(args) + + global logger # pylint: disable=W0603 + logger = logging.getLogger("munet") + + config = parser.get_config(args.config) + logger.info("Loaded config from %s", config["config_pathname"]) + if not config["topology"]["nodes"]: + logger.critical("No nodes defined in config file") + return 1 + + if not args.no_kill: + cleanup_previous() + + loop = None + status = 4 + try: + parser.validate_config(config, logger, args) + if args.validate_only: + return 0 + # Executes the cmd for each node. + loop = get_event_loop() + status = loop.run_until_complete(async_main(args, config)) + except KeyboardInterrupt: + logger.info("Exiting, received KeyboardInterrupt in main") + except Exception as error: + logger.info("Exiting, unexpected exception %s", error, exc_info=True) + finally: + if loop: + loop.close() + + return status + + +if __name__ == "__main__": + exit_status = main() + sys.exit(exit_status) diff --git a/tests/topotests/munet/base.py b/tests/topotests/munet/base.py new file mode 100644 index 0000000..06ca4de --- /dev/null +++ b/tests/topotests/munet/base.py @@ -0,0 +1,3111 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# July 9 2021, Christian Hopps <chopps@labn.net> +# +# Copyright 2021, LabN Consulting, L.L.C. +# +"""A module that implements core functionality for library or standalone use.""" +import asyncio +import datetime +import errno +import ipaddress +import logging +import os +import platform +import re +import readline +import shlex +import signal +import subprocess +import sys +import tempfile +import time as time_mod + +from collections import defaultdict +from pathlib import Path +from typing import Union + +from . import config as munet_config +from . import linux + + +try: + import pexpect + + from pexpect.fdpexpect import fdspawn + from pexpect.popen_spawn import PopenSpawn + + have_pexpect = True +except ImportError: + have_pexpect = False + +PEXPECT_PROMPT = "PEXPECT_PROMPT>" +PEXPECT_CONTINUATION_PROMPT = "PEXPECT_PROMPT+" + +root_hostname = subprocess.check_output("hostname") +our_pid = os.getpid() + + +detailed_cmd_logging = False + + +class MunetError(Exception): + """A generic munet error.""" + + +class CalledProcessError(subprocess.CalledProcessError): + """Improved logging subclass of subprocess.CalledProcessError.""" + + def __str__(self): + o = self.output.strip() if self.output else "" + e = self.stderr.strip() if self.stderr else "" + s = f"returncode: {self.returncode} command: {self.cmd}" + o = "\n\tstdout: " + o if o else "" + e = "\n\tstderr: " + e if e else "" + return s + o + e + + def __repr__(self): + o = self.output.strip() if self.output else "" + e = self.stderr.strip() if self.stderr else "" + return f"munet.base.CalledProcessError({self.returncode}, {self.cmd}, {o}, {e})" + + +class Timeout: + """An object to passively monitor for timeouts.""" + + def __init__(self, delta): + self.delta = datetime.timedelta(seconds=delta) + self.started_on = datetime.datetime.now() + self.expires_on = self.started_on + self.delta + + def elapsed(self): + elapsed = datetime.datetime.now() - self.started_on + return elapsed.total_seconds() + + def is_expired(self): + return datetime.datetime.now() > self.expires_on + + def remaining(self): + remaining = self.expires_on - datetime.datetime.now() + return remaining.total_seconds() + + def __iter__(self): + return self + + def __next__(self): + remaining = self.remaining() + if remaining <= 0: + raise StopIteration() + return remaining + + +def fsafe_name(name): + return "".join(x if x.isalnum() else "_" for x in name) + + +def indent(s): + return "\t" + s.replace("\n", "\n\t") + + +def shell_quote(command): + """Return command wrapped in single quotes.""" + if sys.version_info[0] >= 3: + return shlex.quote(command) + return "'" + command.replace("'", "'\"'\"'") + "'" + + +def cmd_error(rc, o, e): + s = f"rc {rc}" + o = "\n\tstdout: " + o.strip() if o and o.strip() else "" + e = "\n\tstderr: " + e.strip() if e and e.strip() else "" + return s + o + e + + +def shorten(s): + s = s.strip() + i = s.find("\n") + if i > 0: + s = s[: i - 1] + if not s.endswith("..."): + s += "..." + if len(s) > 72: + s = s[:69] + if not s.endswith("..."): + s += "..." + return s + + +def comm_result(rc, o, e): + s = f"\n\treturncode {rc}" if rc else "" + o = "\n\tstdout: " + shorten(o) if o and o.strip() else "" + e = "\n\tstderr: " + shorten(e) if e and e.strip() else "" + return s + o + e + + +def proc_str(p): + if hasattr(p, "args"): + args = p.args if isinstance(p.args, str) else " ".join(p.args) + else: + args = "" + return f"proc pid: {p.pid} args: {args}" + + +def proc_error(p, o, e): + if hasattr(p, "args"): + args = p.args if isinstance(p.args, str) else " ".join(p.args) + else: + args = "" + + s = f"rc {p.returncode} pid {p.pid}" + a = "\n\targs: " + args if args else "" + o = "\n\tstdout: " + (o.strip() if o and o.strip() else "*empty*") + e = "\n\tstderr: " + (e.strip() if e and e.strip() else "*empty*") + return s + a + o + e + + +def comm_error(p): + rc = p.poll() + assert rc is not None + if not hasattr(p, "saved_output"): + p.saved_output = p.communicate() + return proc_error(p, *p.saved_output) + + +async def acomm_error(p): + rc = p.returncode + assert rc is not None + if not hasattr(p, "saved_output"): + p.saved_output = await p.communicate() + return proc_error(p, *p.saved_output) + + +def get_kernel_version(): + kvs = ( + subprocess.check_output("uname -r", shell=True, text=True).strip().split("-", 1) + ) + kv = kvs[0].split(".") + kv = [int(x) for x in kv] + return kv + + +def convert_number(value) -> int: + """Convert a number value with a possible suffix to an integer. + + >>> convert_number("100k") == 100 * 1024 + True + >>> convert_number("100M") == 100 * 1000 * 1000 + True + >>> convert_number("100Gi") == 100 * 1024 * 1024 * 1024 + True + >>> convert_number("55") == 55 + True + """ + if value is None: + raise ValueError("Invalid value None for convert_number") + rate = str(value) + base = 1000 + if rate[-1] == "i": + base = 1024 + rate = rate[:-1] + suffix = "KMGTPEZY" + index = suffix.find(rate[-1]) + if index == -1: + base = 1024 + index = suffix.lower().find(rate[-1]) + if index != -1: + rate = rate[:-1] + return int(rate) * base ** (index + 1) + + +def is_file_like(fo): + return isinstance(fo, int) or hasattr(fo, "fileno") + + +def get_tc_bits_value(user_value): + value = convert_number(user_value) / 1000 + return f"{value:03f}kbit" + + +def get_tc_bytes_value(user_value): + # Raw numbers are bytes in tc + return convert_number(user_value) + + +def get_tmp_dir(uniq): + return os.path.join(tempfile.mkdtemp(), uniq) + + +async def _async_get_exec_path(binary, cmdf, cache): + if isinstance(binary, str): + bins = [binary] + else: + bins = binary + for b in bins: + if b in cache: + return cache[b] + + rc, output, _ = await cmdf("which " + b, warn=False) + if not rc: + cache[b] = os.path.abspath(output.strip()) + return cache[b] + return None + + +def _get_exec_path(binary, cmdf, cache): + if isinstance(binary, str): + bins = [binary] + else: + bins = binary + for b in bins: + if b in cache: + return cache[b] + + rc, output, _ = cmdf("which " + b, warn=False) + if not rc: + cache[b] = os.path.abspath(output.strip()) + return cache[b] + return None + + +def get_event_loop(): + """Configure and return our non-thread using event loop. + + This function configures a new child watcher to not use threads. + Threads cannot be used when we inline unshare a PID namespace. + """ + policy = asyncio.get_event_loop_policy() + loop = policy.get_event_loop() + owatcher = policy.get_child_watcher() + logging.debug( + "event_loop_fixture: global policy %s, current loop %s, current watcher %s", + policy, + loop, + owatcher, + ) + + policy.set_child_watcher(None) + owatcher.close() + + try: + watcher = asyncio.PidfdChildWatcher() # pylint: disable=no-member + except Exception: + watcher = asyncio.SafeChildWatcher() + loop = policy.get_event_loop() + + logging.debug( + "event_loop_fixture: attaching new watcher %s to loop and setting in policy", + watcher, + ) + watcher.attach_loop(loop) + policy.set_child_watcher(watcher) + policy.set_event_loop(loop) + assert asyncio.get_event_loop_policy().get_child_watcher() is watcher + + return loop + + +class Commander: # pylint: disable=R0904 + """An object that can execute commands.""" + + tmux_wait_gen = 0 + + def __init__(self, name, logger=None, unet=None, **kwargs): + """Create a Commander. + + Args: + name: name of the commander object + logger: logger to use for logging commands a defualt is supplied if this + is None + unet: unet that owns this object, only used by Commander in run_in_window, + otherwise can be None. + """ + # del kwargs # deal with lint warning + # logging.warning("Commander: name %s kwargs %s", name, kwargs) + + self.name = name + self.unet = unet + self.deleting = False + self.last = None + self.exec_paths = {} + + if not logger: + logname = f"munet.{self.__class__.__name__.lower()}.{name}" + self.logger = logging.getLogger(logname) + self.logger.setLevel(logging.DEBUG) + else: + self.logger = logger + + super().__init__(**kwargs) + + @property + def is_vm(self): + return False + + @property + def is_container(self): + return False + + def set_logger(self, logfile): + self.logger = logging.getLogger(__name__ + ".commander." + self.name) + self.logger.setLevel(logging.DEBUG) + if isinstance(logfile, str): + handler = logging.FileHandler(logfile, mode="w") + else: + handler = logging.StreamHandler(logfile) + + fmtstr = "%(asctime)s.%(msecs)03d %(levelname)s: {}({}): %(message)s".format( + self.__class__.__name__, self.name + ) + handler.setFormatter(logging.Formatter(fmt=fmtstr)) + self.logger.addHandler(handler) + + def _get_pre_cmd(self, use_str, use_pty, **kwargs): + """Get the pre-user-command values. + + The values returned here should be what is required to cause the user's command + to execute in the correct context (e.g., namespace, container, sshremote). + """ + del kwargs + del use_pty + return "" if use_str else [] + + def __str__(self): + return f"{self.__class__.__name__}({self.name})" + + async def async_get_exec_path(self, binary): + """Return the full path to the binary executable. + + `binary` :: binary name or list of binary names + """ + return await _async_get_exec_path( + binary, self.async_cmd_status_nsonly, self.exec_paths + ) + + def get_exec_path(self, binary): + """Return the full path to the binary executable. + + `binary` :: binary name or list of binary names + """ + return _get_exec_path(binary, self.cmd_status_nsonly, self.exec_paths) + + def get_exec_path_host(self, binary): + """Return the full path to the binary executable. + + If the object is actually a derived class (e.g., a container) this method will + return the exec path for the native namespace rather than the container. The + path is the one which the other xxx_host methods will use. + + `binary` :: binary name or list of binary names + """ + return get_exec_path_host(binary) + + def test(self, flags, arg): + """Run test binary, with flags and arg.""" + test_path = self.get_exec_path(["test"]) + rc, _, _ = self.cmd_status([test_path, flags, arg], warn=False) + return not rc + + def test_nsonly(self, flags, arg): + """Run test binary, with flags and arg.""" + test_path = self.get_exec_path(["test"]) + rc, _, _ = self.cmd_status_nsonly([test_path, flags, arg], warn=False) + return not rc + + def path_exists(self, path): + """Check if path exists.""" + return self.test("-e", path) + + async def cleanup_pid(self, pid, kill_pid=None): + """Signal a pid to exit with escalating forcefulness.""" + if kill_pid is None: + kill_pid = pid + + for sn in (signal.SIGHUP, signal.SIGKILL): + self.logger.debug( + "%s: %s %s (wait %s)", self, signal.Signals(sn).name, kill_pid, pid + ) + + os.kill(kill_pid, sn) + + # No need to wait after this. + if sn == signal.SIGKILL: + return + + # try each signal, waiting 15 seconds for exit before advancing + wait_sec = 30 + self.logger.debug("%s: waiting %ss for pid to exit", self, wait_sec) + for _ in Timeout(wait_sec): + try: + status = os.waitpid(pid, os.WNOHANG) + if status == (0, 0): + await asyncio.sleep(0.1) + else: + self.logger.debug("pid %s exited status %s", pid, status) + return + except OSError as error: + if error.errno == errno.ECHILD: + self.logger.debug("%s: pid %s was reaped", self, pid) + else: + self.logger.warning( + "%s: error waiting on pid %s: %s", self, pid, error + ) + return + self.logger.debug("%s: timeout waiting on pid %s to exit", self, pid) + + def _get_sub_args(self, cmd_list, defaults, use_pty=False, ns_only=False, **kwargs): + """Returns pre-command, cmd, and default keyword args.""" + assert not isinstance(cmd_list, str) + + defaults["shell"] = False + pre_cmd_list = self._get_pre_cmd(False, use_pty, ns_only=ns_only, **kwargs) + cmd_list = [str(x) for x in cmd_list] + + # os_env = {k: v for k, v in os.environ.items() if k.startswith("MUNET")} + # env = {**os_env, **(kwargs["env"] if "env" in kwargs else {})} + env = {**(kwargs["env"] if "env" in kwargs else os.environ)} + if "MUNET_NODENAME" not in env: + env["MUNET_NODENAME"] = self.name + kwargs["env"] = env + + defaults.update(kwargs) + + return pre_cmd_list, cmd_list, defaults + + def _common_prologue(self, async_exec, method, cmd, skip_pre_cmd=False, **kwargs): + cmd_list = self._get_cmd_as_list(cmd) + if method == "_spawn": + defaults = { + "encoding": "utf-8", + "codec_errors": "ignore", + } + else: + defaults = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + } + if not async_exec: + defaults["encoding"] = "utf-8" + + pre_cmd_list, cmd_list, defaults = self._get_sub_args( + cmd_list, defaults, **kwargs + ) + + use_pty = kwargs.get("use_pty", False) + if method == "_spawn": + # spawn doesn't take "shell" keyword arg + if "shell" in defaults: + del defaults["shell"] + # this is required to avoid receiving a STOPPED signal on expect! + if not use_pty: + defaults["preexec_fn"] = os.setsid + defaults["env"]["PS1"] = "$ " + + if not detailed_cmd_logging: + pre_cmd_str = shlex.join(pre_cmd_list) if not skip_pre_cmd else "" + if "nsenter" in pre_cmd_str: + self.logger.debug('%s("%s")', method, shlex.join(cmd_list)) + elif pre_cmd_str: + self.logger.debug( + '%s("%s") [precmd: %s]', method, shlex.join(cmd_list), pre_cmd_str + ) + else: + self.logger.debug('%s("%s") [no precmd]', method, shlex.join(cmd_list)) + else: + self.logger.debug( + '%s: %s %s("%s", pre_cmd: "%s" use_pty: %s kwargs: %.120s)', + self, + "XXX" if method == "_spawn" else "", + method, + cmd_list, + pre_cmd_list if not skip_pre_cmd else "", + use_pty, + defaults, + ) + + actual_cmd_list = cmd_list if skip_pre_cmd else pre_cmd_list + cmd_list + return actual_cmd_list, defaults + + async def _async_popen(self, method, cmd, **kwargs): + """Create a new asynchronous subprocess.""" + acmd, kwargs = self._common_prologue(True, method, cmd, **kwargs) + p = await asyncio.create_subprocess_exec(*acmd, **kwargs) + return p, acmd + + def _popen(self, method, cmd, **kwargs): + """Create a subprocess.""" + acmd, kwargs = self._common_prologue(False, method, cmd, **kwargs) + p = subprocess.Popen(acmd, **kwargs) + return p, acmd + + def _fdspawn(self, fo, **kwargs): + defaults = {} + defaults.update(kwargs) + + if "echo" in defaults: + del defaults["echo"] + + if "encoding" not in defaults: + defaults["encoding"] = "utf-8" + if "codec_errors" not in defaults: + defaults["codec_errors"] = "ignore" + encoding = defaults["encoding"] + + self.logger.debug("%s: _fdspawn(%s, kwargs: %s)", self, fo, defaults) + + p = fdspawn(fo, **defaults) + + # We don't have TTY like conversions of LF to CRLF + p.crlf = os.linesep.encode(encoding) + + # we own the socket now detach the file descriptor to keep it from closing + if hasattr(fo, "detach"): + fo.detach() + + return p + + def _spawn(self, cmd, skip_pre_cmd=False, use_pty=False, echo=False, **kwargs): + logging.debug( + '%s: XXX _spawn: cmd "%s" skip_pre_cmd %s use_pty %s echo %s kwargs %s', + self, + cmd, + skip_pre_cmd, + use_pty, + echo, + kwargs, + ) + actual_cmd, defaults = self._common_prologue( + False, "_spawn", cmd, skip_pre_cmd=skip_pre_cmd, use_pty=use_pty, **kwargs + ) + + self.logger.debug( + '%s: XXX %s("%s", use_pty %s echo %s defaults: %s)', + self, + "PopenSpawn" if not use_pty else "pexpect.spawn", + actual_cmd, + use_pty, + echo, + defaults, + ) + + # We don't specify a timeout it defaults to 30s is that OK? + if not use_pty: + p = PopenSpawn(actual_cmd, **defaults) + else: + p = pexpect.spawn(actual_cmd[0], actual_cmd[1:], echo=echo, **defaults) + return p, actual_cmd + + def spawn( + self, + cmd, + spawned_re, + expects=(), + sends=(), + use_pty=False, + logfile=None, + logfile_read=None, + logfile_send=None, + trace=None, + **kwargs, + ): + """Create a spawned send/expect process. + + Args: + cmd: list of args to exec/popen with, or an already open socket + spawned_re: what to look for to know when done, `spawn` returns when seen + expects: a list of regex other than `spawned_re` to look for. Commonly, + "ogin:" or "[Pp]assword:"r. + sends: what to send when an element of `expects` matches. So e.g., the + username or password if thats what corresponding expect matched. Can + be the empty string to send nothing. + use_pty: true for pty based expect, otherwise uses popen (pipes/files) + trace: if true then log send/expects + **kwargs - kwargs passed on the _spawn. + + Returns: + A pexpect process. + + Raises: + pexpect.TIMEOUT, pexpect.EOF as documented in `pexpect` + CalledProcessError if EOF is seen and `cmd` exited then + raises a CalledProcessError to indicate the failure. + """ + if is_file_like(cmd): + assert not use_pty + ac = "*socket*" + p = self._fdspawn(cmd, **kwargs) + else: + p, ac = self._spawn(cmd, use_pty=use_pty, **kwargs) + + if logfile: + p.logfile = logfile + if logfile_read: + p.logfile_read = logfile_read + if logfile_send: + p.logfile_send = logfile_send + + # for spawned shells (i.e., a direct command an not a console) + # this is wrong and will cause 2 prompts + if not use_pty: + # This isn't very nice looking + p.echo = False + if not is_file_like(cmd): + p.isalive = lambda: p.proc.poll() is None + if not hasattr(p, "close"): + p.close = p.wait + + # Do a quick check to see if we got the prompt right away, otherwise we may be + # at a console so we send a \n to re-issue the prompt + index = p.expect([spawned_re, pexpect.TIMEOUT, pexpect.EOF], timeout=0.1) + if index == 0: + assert p.match is not None + self.logger.debug( + "%s: got spawned_re quick: '%s' matching '%s'", + self, + p.match.group(0), + spawned_re, + ) + return p + + # Now send a CRLF to cause the prompt (or whatever else) to re-issue + p.send("\n") + try: + patterns = [spawned_re, *expects] + + self.logger.debug("%s: expecting: %s", self, patterns) + + while index := p.expect(patterns): + if trace: + assert p.match is not None + self.logger.debug( + "%s: got expect: '%s' matching %d '%s', sending '%s'", + self, + p.match.group(0), + index, + patterns[index], + sends[index - 1], + ) + if sends[index - 1]: + p.send(sends[index - 1]) + + self.logger.debug("%s: expecting again: %s", self, patterns) + self.logger.debug( + "%s: got spawned_re: '%s' matching '%s'", + self, + p.match.group(0), + spawned_re, + ) + return p + except pexpect.TIMEOUT: + self.logger.error( + "%s: TIMEOUT looking for spawned_re '%s' expect buffer so far:\n%s", + self, + spawned_re, + indent(p.buffer), + ) + raise + except pexpect.EOF as eoferr: + if p.isalive(): + raise + rc = p.status + before = indent(p.before) + error = CalledProcessError(rc, ac, output=before) + self.logger.error( + "%s: EOF looking for spawned_re '%s' before EOF:\n%s", + self, + spawned_re, + before, + ) + p.close() + raise error from eoferr + + async def shell_spawn( + self, + cmd, + prompt, + expects=(), + sends=(), + use_pty=False, + will_echo=False, + is_bourne=True, + init_newline=False, + **kwargs, + ): + """Create a shell REPL (read-eval-print-loop). + + Args: + cmd: shell and list of args to popen with, or an already open socket + prompt: the REPL prompt to look for, the function returns when seen + expects: a list of regex other than `spawned_re` to look for. Commonly, + "ogin:" or "[Pp]assword:"r. + sends: what to send when an element of `expects` matches. So e.g., the + username or password if thats what corresponding expect matched. Can + be the empty string to send nothing. + is_bourne: if False then do not modify shell prompt for internal + parser friently format, and do not expect continuation prompts. + init_newline: send an initial newline for non-bourne shell spawns, otherwise + expect the prompt simply from running the command + use_pty: true for pty based expect, otherwise uses popen (pipes/files) + will_echo: bash is buggy in that it echo's to non-tty unlike any other + sh/ksh, set this value to true if running back + **kwargs - kwargs passed on the _spawn. + """ + combined_prompt = r"({}|{})".format(re.escape(PEXPECT_PROMPT), prompt) + + assert not is_file_like(cmd) or not use_pty + p = self.spawn( + cmd, + combined_prompt, + expects=expects, + sends=sends, + use_pty=use_pty, + echo=False, + **kwargs, + ) + assert not p.echo + + if not is_bourne: + if init_newline: + p.send("\n") + return ShellWrapper(p, prompt, will_echo=will_echo) + + ps1 = PEXPECT_PROMPT + ps2 = PEXPECT_CONTINUATION_PROMPT + + # Avoid problems when =/usr/bin/env= prints the values + ps1p = ps1[:5] + "${UNSET_V}" + ps1[5:] + ps2p = ps2[:5] + "${UNSET_V}" + ps2[5:] + + ps1 = re.escape(ps1) + ps2 = re.escape(ps2) + + extra = "PAGER=cat; export PAGER; TERM=dumb; unset HISTFILE; set +o emacs +o vi" + pchg = "PS1='{0}' PS2='{1}' PROMPT_COMMAND=''\n".format(ps1p, ps2p) + p.send(pchg) + return ShellWrapper(p, ps1, ps2, extra_init_cmd=extra, will_echo=will_echo) + + def popen(self, cmd, **kwargs): + """Creates a pipe with the given `command`. + + Args: + cmd: `str` or `list` of command to open a pipe with. + **kwargs: kwargs is eventually passed on to Popen. If `command` is a string + then will be invoked with `bash -c`, otherwise `command` is a list and + will be invoked without a shell. + + Returns: + a subprocess.Popen object. + """ + return self._popen("popen", cmd, **kwargs)[0] + + def popen_nsonly(self, cmd, **kwargs): + """Creates a pipe with the given `command`. + + Args: + cmd: `str` or `list` of command to open a pipe with. + **kwargs: kwargs is eventually passed on to Popen. If `command` is a string + then will be invoked with `bash -c`, otherwise `command` is a list and + will be invoked without a shell. + + Returns: + a subprocess.Popen object. + """ + return self._popen("popen_nsonly", cmd, ns_only=True, **kwargs)[0] + + async def async_popen(self, cmd, **kwargs): + """Creates a pipe with the given `command`. + + Args: + cmd: `str` or `list` of command to open a pipe with. + **kwargs: kwargs is eventually passed on to create_subprocess_exec. If + `command` is a string then will be invoked with `bash -c`, otherwise + `command` is a list and will be invoked without a shell. + + Returns: + a asyncio.subprocess.Process object. + """ + p, _ = await self._async_popen("async_popen", cmd, **kwargs) + return p + + async def async_popen_nsonly(self, cmd, **kwargs): + """Creates a pipe with the given `command`. + + Args: + cmd: `str` or `list` of command to open a pipe with. + **kwargs: kwargs is eventually passed on to create_subprocess_exec. If + `command` is a string then will be invoked with `bash -c`, otherwise + `command` is a list and will be invoked without a shell. + + Returns: + a asyncio.subprocess.Process object. + """ + p, _ = await self._async_popen( + "async_popen_nsonly", cmd, ns_only=True, **kwargs + ) + return p + + async def async_cleanup_proc(self, p, pid=None): + """Terminate a process started with a popen call. + + Args: + p: return value from :py:`async_popen`, :py:`popen`, et al. + pid: pid to signal instead of p.pid, typically a child of + cmd_p == nsenter. + + Returns: + None on success, the ``p`` if multiple timeouts occur even + after a SIGKILL sent. + """ + if not p: + return None + + if p.returncode is not None: + if isinstance(p, subprocess.Popen): + o, e = p.communicate() + else: + o, e = await p.communicate() + self.logger.debug( + "%s: cmd_p already exited status: %s", self, proc_error(p, o, e) + ) + return None + + if pid is None: + pid = p.pid + + self.logger.debug("%s: terminate process: %s (pid %s)", self, proc_str(p), pid) + try: + # This will SIGHUP and wait a while then SIGKILL and return immediately + await self.cleanup_pid(p.pid, pid) + + # Wait another 2 seconds after the possible SIGKILL above for the + # parent nsenter to cleanup and exit + wait_secs = 2 + if isinstance(p, subprocess.Popen): + o, e = p.communicate(timeout=wait_secs) + else: + o, e = await asyncio.wait_for(p.communicate(), timeout=wait_secs) + self.logger.debug( + "%s: cmd_p exited after kill, status: %s", self, proc_error(p, o, e) + ) + except (asyncio.TimeoutError, subprocess.TimeoutExpired): + self.logger.warning("%s: SIGKILL timeout", self) + return p + except Exception as error: + self.logger.warning( + "%s: kill unexpected exception: %s", self, error, exc_info=True + ) + return p + return None + + @staticmethod + def _cmd_status_input(stdin): + pinput = None + if isinstance(stdin, (bytes, str)): + pinput = stdin + stdin = subprocess.PIPE + return pinput, stdin + + def _cmd_status_finish(self, p, c, ac, o, e, raises, warn): + rc = p.returncode + self.last = (rc, ac, c, o, e) + if not rc: + resstr = comm_result(rc, o, e) + if resstr: + self.logger.debug("%s", resstr) + else: + if warn: + self.logger.warning("%s: proc failed: %s", self, proc_error(p, o, e)) + if raises: + # error = Exception("stderr: {}".format(stderr)) + # This annoyingly doesnt' show stderr when printed normally + raise CalledProcessError(rc, ac, o, e) + return rc, o, e + + def _cmd_status(self, cmds, raises=False, warn=True, stdin=None, **kwargs): + """Execute a command.""" + pinput, stdin = Commander._cmd_status_input(stdin) + p, actual_cmd = self._popen("cmd_status", cmds, stdin=stdin, **kwargs) + o, e = p.communicate(pinput) + return self._cmd_status_finish(p, cmds, actual_cmd, o, e, raises, warn) + + async def _async_cmd_status( + self, cmds, raises=False, warn=True, stdin=None, text=None, **kwargs + ): + """Execute a command.""" + pinput, stdin = Commander._cmd_status_input(stdin) + p, actual_cmd = await self._async_popen( + "async_cmd_status", cmds, stdin=stdin, **kwargs + ) + + if text is False: + encoding = None + else: + encoding = kwargs.get("encoding", "utf-8") + + if encoding is not None and isinstance(pinput, str): + pinput = pinput.encode(encoding) + o, e = await p.communicate(pinput) + if encoding is not None: + o = o.decode(encoding) if o is not None else o + e = e.decode(encoding) if e is not None else e + return self._cmd_status_finish(p, cmds, actual_cmd, o, e, raises, warn) + + def _get_cmd_as_list(self, cmd): + """Given a list or string return a list form for execution. + + If `cmd` is a string then the returned list uses bash and looks + like this: ["/bin/bash", "-c", cmd]. Some node types override + this function if they utilize a different shell as to return + a different list of values. + + Args: + cmd: list or string representing the command to execute. + + Returns: + list of commands to execute. + """ + if not isinstance(cmd, str): + cmds = cmd + else: + # Make sure the code doesn't think `cd` will work. + assert not re.match(r"cd(\s*|\s+(\S+))$", cmd) + cmds = ["/bin/bash", "-c", cmd] + return cmds + + def cmd_nostatus(self, cmd, **kwargs): + """Run given command returning output[s]. + + Args: + cmd: `str` or `list` of the command to execute. If a string is given + it is run using a shell, otherwise the list is executed directly + as the binary and arguments. + **kwargs: kwargs is eventually passed on to Popen. If `command` is a string + then will be invoked with `bash -c`, otherwise `command` is a list and + will be invoked without a shell. + + Returns: + if "stderr" is in kwargs and not equal to subprocess.STDOUT, then + both stdout and stderr are returned, otherwise stderr is combined + with stdout and only stdout is returned. + """ + # + # This method serves as the basis for all derived sync cmd variations, so to + # override sync cmd behavior simply override this function and *not* the other + # variations, unless you are changing only that variation's behavior + # + + # XXX change this back to _cmd_status instead of cmd_status when we + # consolidate and cleanup the container overrides of *cmd_* functions + + cmds = cmd + if "stderr" in kwargs and kwargs["stderr"] != subprocess.STDOUT: + _, o, e = self.cmd_status(cmds, **kwargs) + return o, e + if "stderr" in kwargs: + del kwargs["stderr"] + _, o, _ = self.cmd_status(cmds, stderr=subprocess.STDOUT, **kwargs) + return o + + def cmd_status(self, cmd, **kwargs): + """Run given command returning status and outputs. + + Args: + cmd: `str` or `list` of the command to execute. If a string is given + it is run using a shell, otherwise the list is executed directly + as the binary and arguments. + **kwargs: kwargs is eventually passed on to Popen. If `command` is a string + then will be invoked with `bash -c`, otherwise `command` is a list and + will be invoked without a shell. + + Returns: + (status, output, error) are returned + status: the returncode of the command. + output: stdout as a string from the command. + error: stderr as a string from the command. + """ + # + # This method serves as the basis for all derived sync cmd variations, so to + # override sync cmd behavior simply override this function and *not* the other + # variations, unless you are changing only that variation's behavior + # + return self._cmd_status(cmd, **kwargs) + + def cmd_raises(self, cmd, **kwargs): + """Execute a command. Raise an exception on errors. + + Args: + cmd: `str` or `list` of the command to execute. If a string is given + it is run using a shell, otherwise the list is executed directly + as the binary and arguments. + **kwargs: kwargs is eventually passed on to Popen. If `command` is a string + then will be invoked with `bash -c`, otherwise `command` is a list and + will be invoked without a shell. + + Returns: + output: stdout as a string from the command. + + Raises: + CalledProcessError: on non-zero exit status + """ + _, stdout, _ = self._cmd_status(cmd, raises=True, **kwargs) + return stdout + + def cmd_nostatus_nsonly(self, cmd, **kwargs): + # Make sure the command runs on the host and not in any container. + return self.cmd_nostatus(cmd, ns_only=True, **kwargs) + + def cmd_status_nsonly(self, cmd, **kwargs): + # Make sure the command runs on the host and not in any container. + return self._cmd_status(cmd, ns_only=True, **kwargs) + + def cmd_raises_nsonly(self, cmd, **kwargs): + # Make sure the command runs on the host and not in any container. + _, stdout, _ = self._cmd_status(cmd, raises=True, ns_only=True, **kwargs) + return stdout + + async def async_cmd_status(self, cmd, **kwargs): + """Run given command returning status and outputs. + + Args: + cmd: `str` or `list` of the command to execute. If a string is given + it is run using a shell, otherwise the list is executed directly + as the binary and arguments. + **kwargs: kwargs is eventually passed on to create_subprocess_exec. If + `cmd` is a string then will be invoked with `bash -c`, otherwise + `cmd` is a list and will be invoked without a shell. + + Returns: + (status, output, error) are returned + status: the returncode of the command. + output: stdout as a string from the command. + error: stderr as a string from the command. + """ + # + # This method serves as the basis for all derived async cmd variations, so to + # override async cmd behavior simply override this function and *not* the other + # variations, unless you are changing only that variation's behavior + # + return await self._async_cmd_status(cmd, **kwargs) + + async def async_cmd_nostatus(self, cmd, **kwargs): + """Run given command returning output[s]. + + Args: + cmd: `str` or `list` of the command to execute. If a string is given + it is run using a shell, otherwise the list is executed directly + as the binary and arguments. + **kwargs: kwargs is eventually passed on to create_subprocess_exec. If + `cmd` is a string then will be invoked with `bash -c`, otherwise + `cmd` is a list and will be invoked without a shell. + + Returns: + if "stderr" is in kwargs and not equal to subprocess.STDOUT, then + both stdout and stderr are returned, otherwise stderr is combined + with stdout and only stdout is returned. + + """ + # XXX change this back to _async_cmd_status instead of cmd_status when we + # consolidate and cleanup the container overrides of *cmd_* functions + + cmds = cmd + if "stderr" in kwargs and kwargs["stderr"] != subprocess.STDOUT: + _, o, e = await self._async_cmd_status(cmds, **kwargs) + return o, e + if "stderr" in kwargs: + del kwargs["stderr"] + _, o, _ = await self._async_cmd_status(cmds, stderr=subprocess.STDOUT, **kwargs) + return o + + async def async_cmd_raises(self, cmd, **kwargs): + """Execute a command. Raise an exception on errors. + + Args: + cmd: `str` or `list` of the command to execute. If a string is given + it is run using a shell, otherwise the list is executed directly + as the binary and arguments. + **kwargs: kwargs is eventually passed on to create_subprocess_exec. If + `cmd` is a string then will be invoked with `bash -c`, otherwise + `cmd` is a list and will be invoked without a shell. + + Returns: + output: stdout as a string from the command. + + Raises: + CalledProcessError: on non-zero exit status + """ + _, stdout, _ = await self._async_cmd_status(cmd, raises=True, **kwargs) + return stdout + + async def async_cmd_status_nsonly(self, cmd, **kwargs): + # Make sure the command runs on the host and not in any container. + return await self._async_cmd_status(cmd, ns_only=True, **kwargs) + + async def async_cmd_raises_nsonly(self, cmd, **kwargs): + # Make sure the command runs on the host and not in any container. + _, stdout, _ = await self._async_cmd_status( + cmd, raises=True, ns_only=True, **kwargs + ) + return stdout + + def cmd_legacy(self, cmd, **kwargs): + """Execute a command with stdout and stderr joined, *IGNORES ERROR*.""" + defaults = {"stderr": subprocess.STDOUT} + defaults.update(kwargs) + _, stdout, _ = self._cmd_status(cmd, raises=False, **defaults) + return stdout + + # Run a command in a new window (gnome-terminal, screen, tmux, xterm) + def run_in_window( + self, + cmd, + wait_for=False, + background=False, + name=None, + title=None, + forcex=False, + new_window=False, + tmux_target=None, + ns_only=False, + ): + """Run a command in a new window (TMUX, Screen or XTerm). + + Args: + cmd: string to execute. + wait_for: True to wait for exit from command or `str` as channel neme to + signal on exit, otherwise False + background: Do not change focus to new window. + title: Title for new pane (tmux) or window (xterm). + name: Name of the new window (tmux) + forcex: Force use of X11. + new_window: Open new window (instead of pane) in TMUX + tmux_target: Target for tmux pane. + + Returns: + the pane/window identifier from TMUX (depends on `new_window`) + """ + channel = None + if isinstance(wait_for, str): + channel = wait_for + elif wait_for is True: + channel = "{}-wait-{}".format(our_pid, Commander.tmux_wait_gen) + Commander.tmux_wait_gen += 1 + + if forcex or ("TMUX" not in os.environ and "STY" not in os.environ): + root_level = False + else: + root_level = True + + # SUDO: The important thing to note is that with all these methods we are + # executing on the users windowing system, so even though we are normally + # running as root, we will not be when the command is dispatched. Also + # in the case of SCREEN and X11 we need to sudo *back* to the user as well + # This is also done by SSHRemote by defualt so we should *not* sudo back + # if we are SSHRemote. + + # XXX need to test ssh in screen + # XXX need to test ssh in Xterm + sudo_path = get_exec_path_host(["sudo"]) + # This first test case seems same as last but using list instead of string? + if self.is_vm and self.use_ssh: # pylint: disable=E1101 + if isinstance(cmd, str): + cmd = shlex.split(cmd) + cmd = ["/usr/bin/env", f"MUNET_NODENAME={self.name}"] + cmd + + # get the ssh cmd + cmd = self._get_pre_cmd(False, True, ns_only=ns_only) + [shlex.join(cmd)] + unet = self.unet # pylint: disable=E1101 + uns_cmd = unet._get_pre_cmd( # pylint: disable=W0212 + False, True, ns_only=True, root_level=root_level + ) + # get the nsenter for munet + nscmd = [ + sudo_path, + *uns_cmd, + *cmd, + ] + else: + # This is the command to execute to be inside the namespace. + # We are getting into trouble with quoting. + # Why aren't we passing in MUNET_RUNDIR? + cmd = f"/usr/bin/env MUNET_NODENAME={self.name} {cmd}" + # We need sudo b/c we are executing as the user inside the window system. + sudo_path = get_exec_path_host(["sudo"]) + nscmd = ( + sudo_path + + " " + + self._get_pre_cmd(True, True, ns_only=ns_only, root_level=root_level) + + " " + + cmd + ) + + if "TMUX" in os.environ and not forcex: + cmd = [get_exec_path_host("tmux")] + if new_window: + cmd.append("new-window") + cmd.append("-P") + if name: + cmd.append("-n") + cmd.append(name) + if tmux_target: + cmd.append("-t") + cmd.append(tmux_target) + else: + cmd.append("split-window") + cmd.append("-P") + cmd.append("-h") + if not tmux_target: + tmux_target = os.getenv("TMUX_PANE", "") + if background: + cmd.append("-d") + if tmux_target: + cmd.append("-t") + cmd.append(tmux_target) + + # nscmd is always added as single string argument + if not isinstance(nscmd, str): + nscmd = shlex.join(nscmd) + if title: + nscmd = f"printf '\033]2;{title}\033\\'; {nscmd}" + if channel: + nscmd = f'trap "tmux wait -S {channel}; exit 0" EXIT; {nscmd}' + cmd.append(nscmd) + + elif "STY" in os.environ and not forcex: + # wait for not supported in screen for now + channel = None + cmd = [get_exec_path_host("screen")] + if not os.path.exists( + "/run/screen/S-{}/{}".format(os.environ["USER"], os.environ["STY"]) + ): + # XXX not appropriate for ssh + cmd = ["sudo", "-Eu", os.environ["SUDO_USER"]] + cmd + + if title: + cmd.append("-t") + cmd.append(title) + + if isinstance(nscmd, str): + nscmd = shlex.split(nscmd) + cmd.extend(nscmd) + elif "DISPLAY" in os.environ: + cmd = [get_exec_path_host("xterm")] + if "SUDO_USER" in os.environ: + # Do this b/c making things work as root with xauth seems hard + cmd = [ + get_exec_path_host("sudo"), + "-Eu", + os.environ["SUDO_USER"], + ] + cmd + if title: + cmd.append("-T") + cmd.append(title) + + cmd.append("-e") + if isinstance(nscmd, str): + cmd.extend(shlex.split(nscmd)) + else: + cmd.extend(nscmd) + + # if channel: + # return self.cmd_raises(cmd, skip_pre_cmd=True) + # else: + p = commander.popen( + cmd, + # skip_pre_cmd=True, + stdin=None, + shell=False, + ) + # We should reap the child and report the error then. + time_mod.sleep(2) + if p.poll() is not None: + self.logger.error("%s: Failed to launch xterm: %s", self, comm_error(p)) + return p + else: + self.logger.error( + "DISPLAY, STY, and TMUX not in environment, can't open window" + ) + raise Exception("Window requestd but TMUX, Screen and X11 not available") + + # pane_info = self.cmd_raises(cmd, skip_pre_cmd=True, ns_only=True).strip() + # We are prepending the nsenter command, so use unet.rootcmd + pane_info = commander.cmd_raises(cmd).strip() + + # Re-adjust the layout + if "TMUX" in os.environ: + cmd = [ + get_exec_path_host("tmux"), + "select-layout", + "-t", + pane_info if not tmux_target else tmux_target, + "tiled", + ] + commander.cmd_status(cmd) + + # Wait here if we weren't handed the channel to wait for + if channel and wait_for is True: + cmd = [get_exec_path_host("tmux"), "wait", channel] + # commander.cmd_status(cmd, skip_pre_cmd=True) + commander.cmd_status(cmd) + + return pane_info + + def delete(self): + """Calls self.async_delete within an exec loop.""" + asyncio.run(self.async_delete()) + + async def _async_delete(self): + """Delete this objects resources. + + This is the actual implementation of the resource cleanup, each class + should cleanup it's own resources, generally catching and reporting, + but not reraising any exceptions for it's own cleanup, then it should + invoke `super()._async_delete() without catching any exceptions raised + therein. See other examples in `base.py` or `native.py` + """ + self.logger.info("%s: deleted", self) + + async def async_delete(self): + """Delete the Commander (or derived object). + + The actual implementation for any class should be in `_async_delete` + new derived classes should look at the documentation for that function. + """ + try: + self.deleting = True + await self._async_delete() + except Exception as error: + self.logger.error("%s: error while deleting: %s", self, error) + + +class InterfaceMixin: + """A mixin class to support interface functionality.""" + + def __init__(self, *args, **kwargs): + # del kwargs # get rid of lint + # logging.warning("InterfaceMixin: args: %s kwargs: %s", args, kwargs) + + self._intf_addrs = defaultdict(lambda: [None, None]) + self.net_intfs = {} + self.next_intf_index = 0 + self.basename = "eth" + # self.basename = name + "-eth" + super().__init__(*args, **kwargs) + + @property + def intfs(self): + return sorted(self._intf_addrs.keys()) + + @property + def networks(self): + return sorted(self.net_intfs.keys()) + + def get_intf_addr(self, ifname, ipv6=False): + if ifname not in self._intf_addrs: + return None + return self._intf_addrs[ifname][bool(ipv6)] + + def set_intf_addr(self, ifname, ifaddr): + ifaddr = ipaddress.ip_interface(ifaddr) + self._intf_addrs[ifname][ifaddr.version == 6] = ifaddr + + def net_addr(self, netname, ipv6=False): + if netname not in self.net_intfs: + return None + return self.get_intf_addr(self.net_intfs[netname], ipv6=ipv6) + + def set_intf_basename(self, basename): + self.basename = basename + + def get_next_intf_name(self): + while True: + ifname = self.basename + str(self.next_intf_index) + self.next_intf_index += 1 + if ifname not in self._intf_addrs: + break + return ifname + + def get_ns_ifname(self, ifname): + """Return a namespace unique interface name. + + This function is primarily overriden by L3QemuVM, IOW by any class + that doesn't create it's own network namespace and will share that + with the root (unet) namespace. + + Args: + ifname: the interface name. + + Returns: + A name unique to the namespace of this object. By defualt the assumption + is the ifname is namespace unique. + """ + return ifname + + def register_interface(self, ifname): + if ifname not in self._intf_addrs: + self._intf_addrs[ifname] = [None, None] + + def register_network(self, netname, ifname): + if netname in self.net_intfs: + assert self.net_intfs[netname] == ifname + else: + self.net_intfs[netname] = ifname + + def get_linux_tc_args(self, ifname, config): + """Get interface constraints (jitter, delay, rate) for linux TC. + + The keys and their values are as follows: + + delay (int): number of microseconds + jitter (int): number of microseconds + jitter-correlation (float): % correlation to previous (default 10%) + loss (float): % of loss + loss-correlation (float): % correlation to previous (default 0%) + rate (int or str): bits per second, string allows for use of + {KMGTKiMiGiTi} prefixes "i" means K == 1024 otherwise K == 1000 + """ + del ifname # unused + + netem_args = "" + + def get_number(c, v, d=None): + if v not in c or c[v] is None: + return d + return convert_number(c[v]) + + delay = get_number(config, "delay") + if delay is not None: + netem_args += f" delay {delay}usec" + + jitter = get_number(config, "jitter") + if jitter is not None: + if not delay: + raise ValueError("jitter but no delay specified") + jitter_correlation = get_number(config, "jitter-correlation", 10) + netem_args += f" {jitter}usec {jitter_correlation}%" + + loss = get_number(config, "loss") + if loss is not None: + loss_correlation = get_number(config, "loss-correlation", 0) + if loss_correlation: + netem_args += f" loss {loss}% {loss_correlation}%" + else: + netem_args += f" loss {loss}%" + + if (o_rate := config.get("rate")) is None: + return netem_args, "" + + # + # This comment is not correct, but is trying to talk through/learn the + # machinery. + # + # tokens arrive at `rate` into token buffer. + # limit - number of bytes that can be queued waiting for tokens + # -or- + # latency - maximum amount of time a packet may sit in TBF queue + # + # So this just allows receiving faster than rate for latency amount of + # time, before dropping. + # + # latency = sizeofbucket(limit) / rate (peakrate?) + # + # 32kbit + # -------- = latency = 320ms + # 100kbps + # + # -but then- + # burst ([token] buffer) the largest number of instantaneous + # tokens available (i.e, bucket size). + + tbf_args = "" + DEFLIMIT = 1518 * 1 + DEFBURST = 1518 * 2 + try: + tc_rate = o_rate["rate"] + tc_rate = convert_number(tc_rate) + limit = convert_number(o_rate.get("limit", DEFLIMIT)) + burst = convert_number(o_rate.get("burst", DEFBURST)) + except (KeyError, TypeError): + tc_rate = convert_number(o_rate) + limit = convert_number(DEFLIMIT) + burst = convert_number(DEFBURST) + tbf_args += f" rate {tc_rate/1000}kbit" + if delay: + # give an extra 1/10 of buffer space to handle delay + tbf_args += f" limit {limit} burst {burst}" + else: + tbf_args += f" limit {limit} burst {burst}" + + return netem_args, tbf_args + + def set_intf_constraints(self, ifname, **constraints): + """Set interface outbound constraints. + + Set outbound constraints (jitter, delay, rate) for an interface. All arguments + may also be passed as a string and will be converted to numerical format. All + arguments are also optional. If not specified then that existing constraint will + be cleared. + + Args: + ifname: the name of the interface + delay (int): number of microseconds. + jitter (int): number of microseconds. + jitter-correlation (float): Percent correlation to previous (default 10%). + loss (float): Percent of loss. + loss-correlation (float): Percent correlation to previous (default 25%). + rate (int): bits per second, string allows for use of + {KMGTKiMiGiTi} prefixes "i" means K == 1024 otherwise K == 1000. + """ + nsifname = self.get_ns_ifname(ifname) + netem_args, tbf_args = self.get_linux_tc_args(nsifname, constraints) + count = 1 + selector = f"root handle {count}:" + if netem_args: + self.cmd_raises( + f"tc qdisc add dev {nsifname} {selector} netem {netem_args}" + ) + count += 1 + selector = f"parent {count-1}: handle {count}" + # Place rate limit after delay otherwise limit/burst too complex + if tbf_args: + self.cmd_raises(f"tc qdisc add dev {nsifname} {selector} tbf {tbf_args}") + + self.cmd_raises(f"tc qdisc show dev {nsifname}") + + +class LinuxNamespace(Commander, InterfaceMixin): + """A linux Namespace. + + An object that creates and executes commands in a linux namespace + """ + + def __init__( + self, + name, + net=True, + mount=True, + uts=True, + cgroup=False, + ipc=False, + pid=False, + time=False, + user=False, + unshare_inline=False, + set_hostname=True, + private_mounts=None, + **kwargs, + ): + """Create a new linux namespace. + + Args: + name: Internal name for the namespace. + net: Create network namespace. + mount: Create network namespace. + uts: Create UTS (hostname) namespace. + cgroup: Create cgroup namespace. + ipc: Create IPC namespace. + pid: Create PID namespace, also mounts new /proc. + time: Create time namespace. + user: Create user namespace, also keeps capabilities. + set_hostname: Set the hostname to `name`, uts must also be True. + private_mounts: List of strings of the form + "[/external/path:]/internal/path. If no external path is specified a + tmpfs is mounted on the internal path. Any paths specified are first + passed to `mkdir -p`. + unshare_inline: Unshare the process itself rather than using a proxy. + logger: Passed to superclass. + """ + # logging.warning("LinuxNamespace: name %s kwargs %s", name, kwargs) + + super().__init__(name, **kwargs) + + unet = self.unet + + self.logger.debug("%s: creating", self) + + self.cwd = os.path.abspath(os.getcwd()) + + self.nsflags = [] + self.ifnetns = {} + self.uflags = 0 + self.p_ns_fds = None + self.p_ns_fnames = None + self.pid_ns = False + self.init_pid = None + self.unshare_inline = unshare_inline + self.nsenter_fork = True + + # + # Collect the namespaces to unshare + # + if hasattr(self, "proc_path") and self.proc_path: # pylint: disable=no-member + pp = Path(self.proc_path) # pylint: disable=no-member + else: + pp = unet.proc_path if unet else Path("/proc") + pp = pp.joinpath("%P%", "ns") + + flags = "" + uflags = 0 + nslist = [] + nsflags = [] + if cgroup: + nselm = "cgroup" + nslist.append(nselm) + nsflags.append(f"--{nselm}={pp / nselm}") + flags += "C" + uflags |= linux.CLONE_NEWCGROUP + if ipc: + nselm = "ipc" + nslist.append(nselm) + nsflags.append(f"--{nselm}={pp / nselm}") + flags += "i" + uflags |= linux.CLONE_NEWIPC + if mount or pid: + # We need a new mount namespace for pid + nselm = "mnt" + nslist.append(nselm) + nsflags.append(f"--mount={pp / nselm}") + mount = True + flags += "m" + uflags |= linux.CLONE_NEWNS + if net: + nselm = "net" + nslist.append(nselm) + nsflags.append(f"--{nselm}={pp / nselm}") + # if pid: + # os.system(f"touch /tmp/netns-{name}") + # cmd.append(f"--net=/tmp/netns-{name}") + # else: + flags += "n" + uflags |= linux.CLONE_NEWNET + if pid: + self.pid_ns = True + # We look for this b/c the unshare pid will share with /sibn/init + nselm = "pid_for_children" + nslist.append(nselm) + nsflags.append(f"--pid={pp / nselm}") + flags += "p" + uflags |= linux.CLONE_NEWPID + if time: + nselm = "time" + # XXX time_for_children? + nslist.append(nselm) + nsflags.append(f"--{nselm}={pp / nselm}") + flags += "T" + uflags |= linux.CLONE_NEWTIME + if user: + nselm = "user" + nslist.append(nselm) + nsflags.append(f"--{nselm}={pp / nselm}") + flags += "U" + uflags |= linux.CLONE_NEWUSER + if uts: + nselm = "uts" + nslist.append(nselm) + nsflags.append(f"--{nselm}={pp / nselm}") + flags += "u" + uflags |= linux.CLONE_NEWUTS + + assert flags, "LinuxNamespace with no namespaces requested" + + # Should look path up using resources maybe... + mutini_path = get_our_script_path("mutini") + if not mutini_path: + mutini_path = get_our_script_path("mutini.py") + assert mutini_path + cmd = [mutini_path, f"--unshare-flags={flags}", "-v"] + fname = fsafe_name(self.name) + "-mutini.log" + fname = (unet or self).rundir.joinpath(fname) + stdout = open(fname, "w", encoding="utf-8") + stderr = subprocess.STDOUT + + # + # Save the current namespace info to compare against later + # + + if not unet: + nsdict = {x: os.readlink(f"/proc/self/ns/{x}") for x in nslist} + else: + nsdict = { + x: os.readlink(f"{unet.proc_path}/{unet.pid}/ns/{x}") for x in nslist + } + + # + # (A) Basically we need to save the pid of the unshare call for nsenter. + # + # For `unet is not None` (node case) the level this exists at is based on wether + # unet is using a forking nsenter or not. So if unet.nsenter_fork == True then + # we need the child pid of the p.pid (child of pid returned to us), otherwise + # unet.nsenter_fork == False and we just use p.pid as it will be unshare after + # nsenter exec's it. + # + # For the `unet is None` (unet case) the unshare is at the top level or + # non-existent so we always save the returned p.pid. If we are unshare_inline we + # won't have a __pre_cmd but we can save our child_pid to kill later, otherwise + # we set unet.pid to None b/c there's literally nothing to do. + # + # --------------------------------------------------------------------------- + # Breakdown for nested (non-unet) namespace creation, and what PID + # to use for __pre_cmd nsenter use. + # --------------------------------------------------------------------------- + # + # tl;dr + # - for non-inline unshare: Use BBB with pid_for_children, unless none/none + # #then (AAA) returned + # - for inline unshare: use returned pid (AAA) with pid_for_children + # + # All commands use unet.popen to launch the unshare of mutini or cat. + # mutini for PID unshare, otherwise cat. AAA is the returned pid BBB is the + # child of the returned. + # + # Unshare Variant + # --------------- + # + # Here we are running mutini if we are creating new pid namespace workspace, + # cat otherwise. + # + # [PID+PID] pid tree looks like this: + # + # PID NSPID PPID PGID + # uuu - N/A uuu main unet process + # AAA - uuu AAA nsenter (forking, from unet) (in unet namespaces -pid) + # BBB - AAA AAA unshare --fork --kill-child (forking) + # CCC 1 BBB CCC mutini (non-forking since it is pid 1 in new namespace) + # + # Use BBB if we use pid_for_children, CCC for all + # + # [PID+none] For non-pid workspace creation (but unet pid) we use cat and pid + # tree looks like this: + # + # PID PPID PGID + # uuu N/A uuu main unet process + # AAA uuu AAA nsenter (forking) (in unet namespaces -pid) + # BBB AAA AAA unshare -> cat (from unshare non-forking) + # + # Use BBB for all + # + # [none+PID] For pid workspace creation (but NOT unet pid) we use mutini and pid + # tree looks like this: + # + # PID NSPID PPID PGID + # uuu - N/A uuu main unet process + # AAA - uuu AAA nsenter -> unshare --fork --kill-child + # BBB 1 AAA AAA mutini (non-forking since it is pid 1 in new namespace) + # + # Use AAA if we use pid_for_children, BBB for all + # + # [none+none] For non-pid workspace and non-pid unet we use cat and pid tree + # looks like this: + # + # PID PPID PGID + # uuu N/A uuu main unet process + # AAA uuu AAA nsenter -> unshare -> cat + # + # Use AAA for all, there's no BBB + # + # Inline-Unshare Variant + # ---------------------- + # + # For unshare_inline and new PID namespace we have unshared all but our PID + # namespace, but our children end up in the new namespace so the fork popen + # does is good enough. + # + # [PID+PID] pid tree looks like this: + # + # PID NSPID PPID PGID + # uuu - N/A uuu main unet process + # AAA - uuu AAA unshare --fork --kill-child (forking) + # BBB 1 AAA BBB mutini + # + # Use AAA if we use pid_for_children, BBB for all + # + # [PID+none] For non-pid workspace creation (but unet pid) we use cat and pid + # tree looks like this: + # + # PID PPID PGID + # uuu N/A uuu main unet process + # AAA uuu AAA unshare -> cat + # + # Use AAA for all + # + # [none+PID] For pid workspace creation (but NOT unet pid) we use mutini and pid + # tree looks like this: + # + # PID NSPID PPID PGID + # uuu - N/A uuu main unet process + # AAA - uuu AAA unshare --fork --kill-child + # BBB 1 AAA BBB mutini + # + # Use AAA if we use pid_for_children, BBB for all + # + # [none+none] For non-pid workspace and non-pid unet we use cat and pid tree + # looks like this: + # + # PID PPID PGID + # uuu N/A uuu main unet process + # AAA uuu AAA unshare -> cat + # + # Use AAA for all. + # + # + # --------------------------------------------------------------------------- + # Breakdown for unet namespace creation, and what PID to use for __pre_cmd + # --------------------------------------------------------------------------- + # + # tl;dr: save returned PID or nothing. + # - for non-inline unshare: Use AAA with pid_for_children (returned pid) + # - for inline unshare: no __precmd as the fork in popen is enough. + # + # Use commander to launch the unshare mutini/cat (for PID/none + # workspace PID) for non-inline case. AAA is the returned pid BBB is the child + # of the returned. + # + # Unshare Variant + # --------------- + # + # Here we are running mutini if we are creating new pid namespace workspace, + # cat otherwise. + # + # [PID] for unet pid creation pid tree looks like this: + # + # PID NSPID PPID PGID + # uuu - N/A uuu main unet process + # AAA - uuu AAA unshare --fork --kill-child (forking) + # BBB 1 AAA BBB mutini + # + # Use AAA if we use pid_for_children, BBB for all + # + # [none] for unet non-pid, pid tree looks like this: + # + # PID PPID PGID + # uuu N/A uuu main unet process + # AAA uuu AAA unshare -> cat + # + # Use AAA for all + # + # Inline-Unshare Variant + # ----------------------- + # + # For unshare_inline and new PID namespace we have unshared all but our PID + # namespace, but our children end up in the new namespace so the fork in popen + # does is good enough. + # + # [PID] for unet pid creation pid tree looks like this: + # + # PID NSPID PPID PGID + # uuu - N/A uuu main unet process + # AAA 1 uuu AAA mutini + # + # Save p / p.pid, but don't configure any nsenter, uneeded. + # + # Use nothing as the fork when doing a popen is enough to be in all the right + # namepsaces. + # + # [none] for unet non-pid, pid tree looks like this: + # + # PID PPID PGID + # uuu N/A uuu main unet process + # + # Nothing, no __pre_cmd. + # + # + + self.ppid = os.getppid() + self.unshare_inline = unshare_inline + if unshare_inline: + assert unet is None + self.uflags = uflags + # + # Open file descriptors for current namespaces for later resotration. + # + try: + kversion = [int(x) for x in platform.release().split("-")[0].split(".")] + kvok = kversion[0] > 5 or (kversion[0] == 5 and kversion[1] >= 8) + except ValueError: + kvok = False + if ( + not kvok + or sys.version_info[0] < 3 + or (sys.version_info[0] == 3 and sys.version_info[1] < 9) + ): + # get list of namespace file descriptors before we unshare + self.p_ns_fds = [] + self.p_ns_fnames = [] + tmpflags = uflags + for i in range(0, 64): + v = 1 << i + if (tmpflags & v) == 0: + continue + tmpflags &= ~v + if v in linux.namespace_files: + path = os.path.join("/proc/self", linux.namespace_files[v]) + if os.path.exists(path): + self.p_ns_fds.append(os.open(path, 0)) + self.p_ns_fnames.append(f"{path} -> {os.readlink(path)}") + self.logger.debug( + "%s: saving old namespace fd %s (%s)", + self, + self.p_ns_fnames[-1], + self.p_ns_fds[-1], + ) + if not tmpflags: + break + else: + self.p_ns_fds = None + self.p_ns_fnames = None + self.ppid_fd = linux.pidfd_open(self.ppid) + + self.logger.debug( + "%s: unshare to new namespaces %s", + self, + linux.clone_flag_string(uflags), + ) + + linux.unshare(uflags) + + if not pid: + p = None + self.pid = None + self.nsenter_fork = False + else: + # Need to fork to create the PID namespace, but we need to continue + # running from the parent so that things like pytest work. We'll execute + # a mutini process to manage the child init 1 duties. + # + # We (the parent pid) can no longer create threads, due to that being + # restricted by the kernel. See EINVAL in clone(2). + # + p = commander.popen( + [mutini_path, "-v"], + stdin=subprocess.PIPE, + stdout=stdout, + stderr=stderr, + text=True, + # new session/pgid so signals don't propagate + start_new_session=True, + shell=False, + ) + self.pid = p.pid + self.nsenter_fork = False + else: + # Using cat and a stdin PIPE is nice as it will exit when we do. However, + # we also detach it from the pgid so that signals do not propagate to it. + # This is b/c it would exit early (e.g., ^C) then, at least the main munet + # proc which has no other processes like frr daemons running, will take the + # main network namespace with it, which will remove the bridges and the + # veth pair (because the bridge side veth is deleted). + self.logger.debug("%s: creating namespace process: %s", self, cmd) + + # Use the parent unet process if we have one this will cause us to inherit + # the namespaces correctly even in the non-inline case. + parent = self.unet if self.unet else commander + + p = parent.popen( + cmd, + stdin=subprocess.PIPE, + stdout=stdout, + stderr=stderr, + text=True, + start_new_session=not unet, + shell=False, + ) + + # The pid number returned is in the global pid namespace. For unshare_inline + # this can be unfortunate b/c our /proc has been remounted in our new pid + # namespace and won't contain global pid namespace pids. To solve for this + # we get all the pid values for the process below. + + # See (A) above for when we need the child pid. + self.logger.debug("%s: namespace process: %s", self, proc_str(p)) + self.pid = p.pid + if unet and unet.nsenter_fork: + assert not unet.unshare_inline + # Need child pid of p.pid + pgrep = unet.rootcmd.get_exec_path("pgrep") + # a sing fork was done + child_pid = unet.rootcmd.cmd_raises([pgrep, "-o", "-P", str(p.pid)]) + self.pid = int(child_pid.strip()) + self.logger.debug("%s: child of namespace process: %s", self, pid) + + self.p = p + + # Let's always have a valid value. + if self.pid is None: + self.pid = our_pid + + # + # Let's find all our pids in the nested PID namespaces + # + if unet: + proc_path = unet.proc_path + else: + proc_path = self.proc_path if hasattr(self, "proc_path") else "/proc" + proc_path = f"{proc_path}/{self.pid}" + + pid_status = open(f"{proc_path}/status", "r", encoding="ascii").read() + m = re.search(r"\nNSpid:((?:\t[0-9]+)+)\n", pid_status) + self.pids = [int(x) for x in m.group(1).strip().split("\t")] + assert self.pids[0] == self.pid + + self.logger.debug("%s: namespace scoped pids: %s", self, self.pids) + + # ----------------------------------------------- + # Now let's wait until unshare completes it's job + # ----------------------------------------------- + timeout = Timeout(30) + if self.pid is not None and self.pid != our_pid: + while (not p or not p.poll()) and not timeout.is_expired(): + # check new namespace values against old (nsdict), unshare + # can actually take a bit to complete. + for fname in tuple(nslist): + # self.pid will be the global pid b/c we didn't unshare_inline + nspath = f"{proc_path}/ns/{fname}" + try: + nsf = os.readlink(nspath) + except OSError as error: + self.logger.debug( + "unswitched: error (ok) checking %s: %s", nspath, error + ) + continue + if nsdict[fname] != nsf: + self.logger.debug( + "switched: original %s current %s", nsdict[fname], nsf + ) + nslist.remove(fname) + elif unshare_inline: + logging.warning( + "unshare_inline not unshared %s == %s", nsdict[fname], nsf + ) + else: + self.logger.debug( + "unswitched: current %s elapsed: %s", nsf, timeout.elapsed() + ) + if not nslist: + self.logger.debug( + "all done waiting for unshare after %s", timeout.elapsed() + ) + break + + elapsed = int(timeout.elapsed()) + if elapsed <= 3: + time_mod.sleep(0.1) + else: + self.logger.info( + "%s: unshare taking more than %ss: %s", self, elapsed, nslist + ) + time_mod.sleep(1) + + if p is not None and p.poll(): + self.logger.error("%s: namespace process failed: %s", self, comm_error(p)) + assert p.poll() is None, "unshare failed" + + # + # Setup the pre-command to enter the target namespace from the running munet + # process using self.pid + # + + if pid: + nsenter_fork = True + elif unet and unet.nsenter_fork: + # if unet created a pid namespace we need to enter it since we aren't + # entering a child pid namespace we created for the node. Otherwise + # we have a /proc remounted under unet, but our process is running in + # the root pid namepsace + nselm = "pid_for_children" + nsflags.append(f"--pid={pp / nselm}") + nsenter_fork = True + else: + # We dont need a fork. + nsflags.append("-F") + nsenter_fork = False + + # Save nsenter values if running from root namespace + # we need this for the unshare_inline case when run externally (e.g., from + # within tmux server). + root_nsflags = [x.replace("%P%", str(self.pid)) for x in nsflags] + self.__root_base_pre_cmd = ["/usr/bin/nsenter", *root_nsflags] + self.__root_pre_cmd = list(self.__root_base_pre_cmd) + + if unshare_inline: + assert unet is None + # We have nothing to do here since our process is now in the correct + # namespaces and children will inherit from us, even the PID namespace will + # be corrent b/c commands are run by first forking. + self.nsenter_fork = False + self.nsflags = [] + self.__base_pre_cmd = [] + else: + # We will use nsenter + self.nsenter_fork = nsenter_fork + self.nsflags = nsflags + self.__base_pre_cmd = list(self.__root_base_pre_cmd) + + self.__pre_cmd = list(self.__base_pre_cmd) + + # Always mark new mount namespaces as recursive private + if mount: + # if self.p is None and not pid: + self.cmd_raises_nsonly("mount --make-rprivate /") + + # We need to remount the procfs for the new PID namespace, since we aren't using + # unshare(1) which does that for us. + if pid and unshare_inline: + assert mount + self.cmd_raises_nsonly("mount -t proc proc /proc") + + # We do not want cmd_status in child classes (e.g., container) for + # the remaining setup calls in this __init__ function. + + if net: + # Remount /sys to pickup any changes in the network, but keep root + # /sys/fs/cgroup. This pattern could be made generic and supported for any + # overlapping mounts + if mount: + tmpmnt = f"/tmp/cgm-{self.pid}" + self.cmd_status_nsonly( + f"mkdir {tmpmnt} && mount --rbind /sys/fs/cgroup {tmpmnt}" + ) + rc = o = e = None + for i in range(0, 10): + rc, o, e = self.cmd_status_nsonly( + "mount -t sysfs sysfs /sys", warn=False + ) + if not rc: + break + self.logger.debug( + "got error mounting new sysfs will retry: %s", + cmd_error(rc, o, e), + ) + time_mod.sleep(1) + else: + raise Exception(cmd_error(rc, o, e)) + + self.cmd_status_nsonly( + f"mount --move {tmpmnt} /sys/fs/cgroup && rmdir {tmpmnt}" + ) + + # Original micronet code + # self.cmd_raises_nsonly("mount -t sysfs sysfs /sys") + # self.cmd_raises_nsonly( + # "mount -o rw,nosuid,nodev,noexec,relatime " + # "-t cgroup2 cgroup /sys/fs/cgroup" + # ) + + # Set the hostname to the namespace name + if uts and set_hostname: + self.cmd_status_nsonly("hostname " + self.name) + nroot = subprocess.check_output("hostname") + if unshare_inline or (unet and unet.unshare_inline): + assert ( + root_hostname != nroot + ), f'hostname unchanged from "{nroot}" wanted "{self.name}"' + else: + # Assert that we didn't just change the host hostname + assert ( + root_hostname == nroot + ), f'root hostname "{root_hostname}" changed to "{nroot}"!' + + if private_mounts: + if isinstance(private_mounts, str): + private_mounts = [private_mounts] + for m in private_mounts: + s = m.split(":", 1) + if len(s) == 1: + self.tmpfs_mount(s[0]) + else: + self.bind_mount(s[0], s[1]) + + # this will fail if running inside the namespace with PID + if pid: + o = self.cmd_nostatus_nsonly("ls -l /proc/1/ns") + else: + o = self.cmd_nostatus_nsonly("ls -l /proc/self/ns") + + self.logger.debug("namespaces:\n %s", o) + + # will cache the path, which is important in delete to avoid running a shell + # which can hang during cleanup + self.ip_path = get_exec_path_host("ip") + if net: + self.cmd_status_nsonly([self.ip_path, "link", "set", "lo", "up"]) + + self.logger.info("%s: created", self) + + def _get_pre_cmd(self, use_str, use_pty, ns_only=False, root_level=False, **kwargs): + """Get the pre-user-command values. + + The values returned here should be what is required to cause the user's command + to execute in the correct context (e.g., namespace, container, sshremote). + """ + del kwargs + del ns_only + del use_pty + pre_cmd = self.__root_pre_cmd if root_level else self.__pre_cmd + return shlex.join(pre_cmd) if use_str else list(pre_cmd) + + def tmpfs_mount(self, inner): + self.logger.debug("Mounting tmpfs on %s", inner) + self.cmd_raises("mkdir -p " + inner) + self.cmd_raises("mount -n -t tmpfs tmpfs " + inner) + + def bind_mount(self, outer, inner): + self.logger.debug("Bind mounting %s on %s", outer, inner) + if commander.test("-f", outer): + self.cmd_raises(f"mkdir -p {os.path.dirname(inner)} && touch {inner}") + else: + if not commander.test("-e", outer): + commander.cmd_raises_nsonly(f"mkdir -p {outer}") + self.cmd_raises(f"mkdir -p {inner}") + self.cmd_raises("mount --rbind {} {} ".format(outer, inner)) + + def add_netns(self, ns): + self.logger.debug("Adding network namespace %s", ns) + + if os.path.exists("/run/netns/{}".format(ns)): + self.logger.warning("%s: Removing existing nsspace %s", self, ns) + try: + self.delete_netns(ns) + except Exception as ex: + self.logger.warning( + "%s: Couldn't remove existing nsspace %s: %s", + self, + ns, + str(ex), + exc_info=True, + ) + self.cmd_raises_nsonly([self.ip_path, "netns", "add", ns]) + + def delete_netns(self, ns): + self.logger.debug("Deleting network namespace %s", ns) + self.cmd_raises_nsonly([self.ip_path, "netns", "delete", ns]) + + def set_intf_netns(self, intf, ns, up=False): + # In case a user hard-codes 1 thinking it "resets" + ns = str(ns) + if ns == "1": + ns = str(self.pid) + + self.logger.debug("Moving interface %s to namespace %s", intf, ns) + + cmd = [self.ip_path, "link", "set", intf, "netns", ns] + if up: + cmd.append("up") + self.intf_ip_cmd(intf, cmd) + if ns == str(self.pid): + # If we are returning then remove from dict + if intf in self.ifnetns: + del self.ifnetns[intf] + else: + self.ifnetns[intf] = ns + + def reset_intf_netns(self, intf): + self.logger.debug("Moving interface %s to default namespace", intf) + self.set_intf_netns(intf, str(self.pid)) + + def intf_ip_cmd(self, intf, cmd): + """Run an ip command, considering an interface's possible namespace.""" + if intf in self.ifnetns: + if isinstance(cmd, list): + assert cmd[0].endswith("ip") + cmd[1:1] = ["-n", self.ifnetns[intf]] + else: + assert cmd.startswith("ip ") + cmd = "ip -n " + self.ifnetns[intf] + cmd[2:] + self.cmd_raises_nsonly(cmd) + + def intf_tc_cmd(self, intf, cmd): + """Run a tc command, considering an interface's possible namespace.""" + if intf in self.ifnetns: + if isinstance(cmd, list): + assert cmd[0].endswith("tc") + cmd[1:1] = ["-n", self.ifnetns[intf]] + else: + assert cmd.startswith("tc ") + cmd = "tc -n " + self.ifnetns[intf] + cmd[2:] + self.cmd_raises_nsonly(cmd) + + def set_ns_cwd(self, cwd: Union[str, Path]): + """Common code for changing pre_cmd and pre_nscmd.""" + self.logger.debug("%s: new CWD %s", self, cwd) + self.__root_pre_cmd = self.__root_base_pre_cmd + ["--wd=" + str(cwd)] + if self.__pre_cmd: + self.__pre_cmd = self.__base_pre_cmd + ["--wd=" + str(cwd)] + elif self.unshare_inline: + os.chdir(cwd) + + async def _async_delete(self): + if type(self) == LinuxNamespace: # pylint: disable=C0123 + self.logger.info("%s: deleting", self) + else: + self.logger.debug("%s: LinuxNamespace sub-class deleting", self) + + # Signal pid namespace proc to exit + if ( + (self.p is None or self.p.pid != self.pid) + and self.pid + and self.pid != our_pid + ): + self.logger.debug( + "cleanup pid on separate pid %s from proc pid %s", + self.pid, + self.p.pid if self.p else None, + ) + await self.cleanup_pid(self.pid) + + if self.p is not None: + self.logger.debug("cleanup proc pid %s", self.p.pid) + await self.async_cleanup_proc(self.p) + + # return to the previous namespace, need to do this in case anothe munet + # is being created, especially when it plans to inherit the parent's (host) + # namespace. + if self.uflags: + logging.info("restoring from inline unshare: cwd: %s", os.getcwd()) + # This only works in linux>=5.8 + if self.p_ns_fds is None: + self.logger.debug( + "%s: restoring namespaces %s", + self, + linux.clone_flag_string(self.uflags), + ) + # fd = linux.pidfd_open(self.ppid) + fd = self.ppid_fd + retry = 3 + for i in range(0, retry): + try: + linux.setns(fd, self.uflags) + except OSError as error: + self.logger.warning( + "%s: could not reset to old namespace fd %s: %s", + self, + fd, + error, + ) + if i == retry - 1: + raise + time_mod.sleep(1) + os.close(fd) + else: + while self.p_ns_fds: + fd = self.p_ns_fds.pop() + fname = self.p_ns_fnames.pop() + self.logger.debug( + "%s: restoring namespace from fd %s (%s)", self, fname, fd + ) + retry = 3 + for i in range(0, retry): + try: + linux.setns(fd, 0) + break + except OSError as error: + self.logger.warning( + "%s: could not reset to old namespace fd %s (%s): %s", + self, + fname, + fd, + error, + ) + if i == retry - 1: + raise + time_mod.sleep(1) + os.close(fd) + self.p_ns_fds = None + self.p_ns_fnames = None + logging.info("restored from unshare: cwd: %s", os.getcwd()) + + self.__root_base_pre_cmd = ["/bin/false"] + self.__base_pre_cmd = ["/bin/false"] + self.__root_pre_cmd = ["/bin/false"] + self.__pre_cmd = ["/bin/false"] + + await super()._async_delete() + + +class SharedNamespace(Commander): + """Share another namespace. + + An object that executes commands in an existing pid's linux namespace + """ + + def __init__(self, name, pid=None, nsflags=None, **kwargs): + """Share a linux namespace. + + Args: + name: Internal name for the namespace. + pid: PID of the process to share with. + nsflags: nsenter flags to pass to inherit namespaces from + """ + super().__init__(name, **kwargs) + + self.logger.debug("%s: Creating", self) + + self.cwd = os.path.abspath(os.getcwd()) + self.pid = pid if pid is not None else our_pid + + nsflags = (x.replace("%P%", str(self.pid)) for x in nsflags) if nsflags else [] + self.__base_pre_cmd = ["/usr/bin/nsenter", *nsflags] if nsflags else [] + self.__pre_cmd = self.__base_pre_cmd + self.ip_path = self.get_exec_path("ip") + + def _get_pre_cmd(self, use_str, use_pty, ns_only=False, root_level=False, **kwargs): + """Get the pre-user-command values. + + The values returned here should be what is required to cause the user's command + to execute in the correct context (e.g., namespace, container, sshremote). + """ + del kwargs + del ns_only + del use_pty + assert not root_level + return shlex.join(self.__pre_cmd) if use_str else list(self.__pre_cmd) + + def set_ns_cwd(self, cwd: Union[str, Path]): + """Common code for changing pre_cmd and pre_nscmd.""" + self.logger.debug("%s: new CWD %s", self, cwd) + self.__pre_cmd = self.__base_pre_cmd + ["--wd=" + str(cwd)] + + +class Bridge(SharedNamespace, InterfaceMixin): + """A linux bridge.""" + + next_ord = 1 + + @classmethod + def _get_next_id(cls): + # Do not use `cls` here b/c that makes the variable class specific + n = Bridge.next_ord + Bridge.next_ord = n + 1 + return n + + def __init__(self, name=None, mtu=None, unet=None, **kwargs): + """Create a linux Bridge.""" + self.id = self._get_next_id() + if not name: + name = "br{}".format(self.id) + + unet_pid = our_pid if unet.pid is None else unet.pid + + super().__init__(name, pid=unet_pid, nsflags=unet.nsflags, unet=unet, **kwargs) + + self.set_intf_basename(self.name + "-e") + + self.mtu = mtu + + self.logger.debug("Bridge: Creating") + + assert len(self.name) <= 16 # Make sure fits in IFNAMSIZE + self.cmd_raises(f"ip link delete {name} || true") + self.cmd_raises(f"ip link add {name} type bridge") + if self.mtu: + self.cmd_raises(f"ip link set {name} mtu {self.mtu}") + self.cmd_raises(f"ip link set {name} up") + + self.logger.debug("%s: Created, Running", self) + + def get_ifname(self, netname): + return self.net_intfs[netname] if netname in self.net_intfs else None + + async def _async_delete(self): + """Stop the bridge (i.e., delete the linux resources).""" + if type(self) == Bridge: # pylint: disable=C0123 + self.logger.info("%s: deleting", self) + else: + self.logger.debug("%s: Bridge sub-class deleting", self) + + rc, o, e = await self.async_cmd_status( + [self.ip_path, "link", "show", self.name], + stdin=subprocess.DEVNULL, + start_new_session=True, + warn=False, + ) + if not rc: + rc, o, e = await self.async_cmd_status( + [self.ip_path, "link", "delete", self.name], + stdin=subprocess.DEVNULL, + start_new_session=True, + warn=False, + ) + if rc: + self.logger.error( + "%s: error deleting bridge %s: %s", + self, + self.name, + cmd_error(rc, o, e), + ) + await super()._async_delete() + + +class BaseMunet(LinuxNamespace): + """Munet.""" + + def __init__( + self, + name="munet", + isolated=True, + pid=True, + rundir=None, + pytestconfig=None, + **kwargs, + ): + """Create a Munet.""" + # logging.warning("BaseMunet: %s", name) + + self.hosts = {} + self.switches = {} + self.links = {} + self.macs = {} + self.rmacs = {} + self.isolated = isolated + + self.cli_server = None + self.cli_sockpath = None + self.cli_histfile = None + self.cli_in_window_cmds = {} + self.cli_run_cmds = {} + + # + # We need a directory for various files + # + if not rundir: + rundir = "/tmp/munet" + self.rundir = Path(rundir) + + # + # Always having a global /proc is required to keep things from exploding + # complexity with nested new pid namespaces.. + # + if pid: + self.proc_path = Path(tempfile.mkdtemp(suffix="-proc", prefix="mu-")) + logging.debug("%s: mounting /proc on proc_path %s", name, self.proc_path) + linux.mount("proc", str(self.proc_path), "proc") + else: + self.proc_path = Path("/proc") + + # + # Now create a root level commander that works regardless of whether we inline + # unshare or not. Save it in the global variable as well + # + + if not self.isolated: + self.rootcmd = commander + elif not pid: + nsflags = ( + f"--mount={self.proc_path / '1/ns/mnt'}", + f"--net={self.proc_path / '1/ns/net'}", + f"--uts={self.proc_path / '1/ns/uts'}", + # f"--ipc={self.proc_path / '1/ns/ipc'}", + # f"--time={self.proc_path / '1/ns/time'}", + # f"--cgroup={self.proc_path / '1/ns/cgroup'}", + ) + self.rootcmd = SharedNamespace("root", pid=1, nsflags=nsflags) + else: + # XXX user + nsflags = ( + # XXX Backing up PID namespace just doesn't work. + # f"--pid={self.proc_path / '1/ns/pid_for_children'}", + f"--mount={self.proc_path / '1/ns/mnt'}", + f"--net={self.proc_path / '1/ns/net'}", + f"--uts={self.proc_path / '1/ns/uts'}", + # f"--ipc={self.proc_path / '1/ns/ipc'}", + # f"--time={self.proc_path / '1/ns/time'}", + # f"--cgroup={self.proc_path / '1/ns/cgroup'}", + ) + self.rootcmd = SharedNamespace("root", pid=1, nsflags=nsflags) + global roothost # pylint: disable=global-statement + + roothost = self.rootcmd + + self.cfgopt = munet_config.ConfigOptionsProxy(pytestconfig) + + super().__init__( + name, mount=True, net=isolated, uts=isolated, pid=pid, unet=None, **kwargs + ) + + # This allows us to cleanup any leftover running munet's + if "MUNET_PID" in os.environ: + if os.environ["MUNET_PID"] != str(our_pid): + logging.error( + "Found env MUNET_PID != our pid %s, instead its %s, changing", + our_pid, + os.environ["MUNET_PID"], + ) + os.environ["MUNET_PID"] = str(our_pid) + + # this is for testing purposes do not use + if not BaseMunet.g_unet: + BaseMunet.g_unet = self + + self.logger.debug("%s: Creating", self) + + def __getitem__(self, key): + if key in self.switches: + return self.switches[key] + return self.hosts[key] + + def add_host(self, name, cls=LinuxNamespace, **kwargs): + """Add a host to munet.""" + self.logger.debug("%s: add_host %s(%s)", self, cls.__name__, name) + + self.hosts[name] = cls(name, unet=self, **kwargs) + + # Create a new mounted FS for tracking nested network namespaces creatd by the + # user with `ip netns add` + + # XXX why is this failing with podman??? + # self.hosts[name].tmpfs_mount("/run/netns") + + return self.hosts[name] + + def add_link(self, node1, node2, if1, if2, mtu=None, **intf_constraints): + """Add a link between switch and node or 2 nodes. + + If constraints are given they are applied to each endpoint. See + `InterfaceMixin::set_intf_constraints()` for more info. + """ + isp2p = False + + try: + name1 = node1.name + except AttributeError: + if node1 in self.switches: + node1 = self.switches[node1] + else: + node1 = self.hosts[node1] + name1 = node1.name + + try: + name2 = node2.name + except AttributeError: + if node2 in self.switches: + node2 = self.switches[node2] + else: + node2 = self.hosts[node2] + name2 = node2.name + + if name1 in self.switches: + assert name2 in self.hosts + elif name2 in self.switches: + assert name1 in self.hosts + name1, name2 = name2, name1 + if1, if2 = if2, if1 + else: + # p2p link + assert name1 in self.hosts + assert name2 in self.hosts + isp2p = True + + lname = "{}:{}-{}:{}".format(name1, if1, name2, if2) + self.logger.debug("%s: add_link %s%s", self, lname, " p2p" if isp2p else "") + self.links[lname] = (name1, if1, name2, if2) + + # And create the veth now. + if isp2p: + lhost, rhost = self.hosts[name1], self.hosts[name2] + lifname = "i1{:x}".format(lhost.pid) + + # Done at root level + nsif1 = lhost.get_ns_ifname(if1) + nsif2 = rhost.get_ns_ifname(if2) + + # Use pids[-1] to get the unet scoped pid for hosts + self.cmd_raises_nsonly( + f"ip link add {lifname} type veth peer name {nsif2}" + f" netns {rhost.pids[-1]}" + ) + self.cmd_raises_nsonly(f"ip link set {lifname} netns {lhost.pids[-1]}") + + lhost.cmd_raises_nsonly("ip link set {} name {}".format(lifname, nsif1)) + if mtu: + lhost.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif1, mtu)) + lhost.cmd_raises_nsonly("ip link set {} up".format(nsif1)) + lhost.register_interface(if1) + + if mtu: + rhost.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif2, mtu)) + rhost.cmd_raises_nsonly("ip link set {} up".format(nsif2)) + rhost.register_interface(if2) + else: + switch = self.switches[name1] + rhost = self.hosts[name2] + + nsif1 = switch.get_ns_ifname(if1) + nsif2 = rhost.get_ns_ifname(if2) + + if mtu is None: + mtu = switch.mtu + + if len(nsif1) > 16: + self.logger.error('"%s" len %s > 16', nsif1, len(nsif1)) + elif len(nsif2) > 16: + self.logger.error('"%s" len %s > 16', nsif2, len(nsif2)) + assert len(nsif1) <= 16 and len(nsif2) <= 16 # Make sure fits in IFNAMSIZE + + self.logger.debug("%s: Creating veth pair for link %s", self, lname) + + # Use pids[-1] to get the unet scoped pid for hosts + # switch is already in our namespace so nothing to convert. + self.cmd_raises_nsonly( + f"ip link add {nsif1} type veth peer name {nsif2}" + f" netns {rhost.pids[-1]}" + ) + + if mtu: + # if switch.mtu: + # # the switch interface should match the switch config + # switch.cmd_raises_nsonly( + # "ip link set {} mtu {}".format(if1, switch.mtu) + # ) + switch.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif1, mtu)) + rhost.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif2, mtu)) + + switch.register_interface(if1) + rhost.register_interface(if2) + rhost.register_network(switch.name, if2) + + switch.cmd_raises_nsonly(f"ip link set {nsif1} master {switch.name}") + + switch.cmd_raises_nsonly(f"ip link set {nsif1} up") + rhost.cmd_raises_nsonly(f"ip link set {nsif2} up") + + # Cache the MAC values, and reverse mapping + self.get_mac(name1, nsif1) + self.get_mac(name2, nsif2) + + # Setup interface constraints if provided + if intf_constraints: + node1.set_intf_constraints(if1, **intf_constraints) + node2.set_intf_constraints(if2, **intf_constraints) + + def add_switch(self, name, cls=Bridge, **kwargs): + """Add a switch to munet.""" + self.logger.debug("%s: add_switch %s(%s)", self, cls.__name__, name) + self.switches[name] = cls(name, unet=self, **kwargs) + return self.switches[name] + + def get_mac(self, name, ifname): + if name in self.hosts: + dev = self.hosts[name] + else: + dev = self.switches[name] + + nsifname = self.get_ns_ifname(ifname) + + if (name, ifname) not in self.macs: + _, output, _ = dev.cmd_status_nsonly("ip -o link show " + nsifname) + m = re.match(".*link/(loopback|ether) ([0-9a-fA-F:]+) .*", output) + mac = m.group(2) + self.macs[(name, ifname)] = mac + self.rmacs[mac] = (name, ifname) + + return self.macs[(name, ifname)] + + async def _delete_link(self, lname): + rname, rif = self.links[lname][2:4] + host = self.hosts[rname] + nsrif = host.get_ns_ifname(rif) + + self.logger.debug("%s: Deleting veth pair for link %s", self, lname) + rc, o, e = await host.async_cmd_status_nsonly( + [self.ip_path, "link", "delete", nsrif], + stdin=subprocess.DEVNULL, + start_new_session=True, + warn=False, + ) + if rc: + self.logger.error("Err del veth pair %s: %s", lname, cmd_error(rc, o, e)) + + async def _delete_links(self): + # for x in self.links: + # await self._delete_link(x) + return await asyncio.gather(*[self._delete_link(x) for x in self.links]) + + async def _async_delete(self): + """Delete the munet topology.""" + # logger = self.logger if False else logging + logger = self.logger + if type(self) == BaseMunet: # pylint: disable=C0123 + logger.info("%s: deleting.", self) + else: + logger.debug("%s: BaseMunet sub-class deleting.", self) + + logger.debug("Deleting links") + try: + await self._delete_links() + except Exception as error: + logger.error("%s: error deleting links: %s", self, error, exc_info=True) + + logger.debug("Deleting hosts and bridges") + try: + # Delete hosts and switches, wait for them all to complete + # even if there is an exception. + htask = [x.async_delete() for x in self.hosts.values()] + stask = [x.async_delete() for x in self.switches.values()] + await asyncio.gather(*htask, *stask, return_exceptions=True) + except Exception as error: + logger.error( + "%s: error deleting hosts and switches: %s", self, error, exc_info=True + ) + + self.links = {} + self.hosts = {} + self.switches = {} + + try: + if self.cli_server: + self.cli_server.cancel() + self.cli_server = None + if self.cli_sockpath: + await self.async_cmd_status( + "rm -rf " + os.path.dirname(self.cli_sockpath) + ) + self.cli_sockpath = None + except Exception as error: + logger.error( + "%s: error cli server or sockpaths: %s", self, error, exc_info=True + ) + + try: + if self.cli_histfile: + readline.write_history_file(self.cli_histfile) + self.cli_histfile = None + except Exception as error: + logger.error( + "%s: error saving history file: %s", self, error, exc_info=True + ) + + # XXX for some reason setns during the delete is changing our dir to /. + cwd = os.getcwd() + + try: + await super()._async_delete() + except Exception as error: + logger.error( + "%s: error deleting parent classes: %s", self, error, exc_info=True + ) + os.chdir(cwd) + + try: + if self.proc_path and str(self.proc_path) != "/proc": + logger.debug("%s: umount, remove proc_path %s", self, self.proc_path) + linux.umount(str(self.proc_path), 0) + os.rmdir(self.proc_path) + except Exception as error: + logger.warning( + "%s: error umount and removing proc_path %s: %s", + self, + self.proc_path, + error, + exc_info=True, + ) + try: + linux.umount(str(self.proc_path), linux.MNT_DETACH) + except Exception as error2: + logger.error( + "%s: error umount with detach proc_path %s: %s", + self, + self.proc_path, + error2, + exc_info=True, + ) + + if BaseMunet.g_unet == self: + BaseMunet.g_unet = None + + +BaseMunet.g_unet = None + +if True: # pylint: disable=using-constant-test + + class ShellWrapper: + """A Read-Execute-Print-Loop (REPL) interface. + + A newline or prompt changing command should be sent to the + spawned child prior to creation as the `prompt` will be `expect`ed + """ + + def __init__( + self, + spawn, + prompt, + continuation_prompt=None, + extra_init_cmd=None, + will_echo=False, + escape_ansi=False, + ): + self.echo = will_echo + self.escape = ( + re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]") if escape_ansi else None + ) + + logging.debug( + 'ShellWraper: XXX prompt "%s" will_echo %s child.echo %s', + prompt, + will_echo, + spawn.echo, + ) + + self.child = spawn + if self.child.echo: + logging.info("Setting child to echo") + self.child.setecho(False) + self.child.waitnoecho() + assert not self.child.echo + + self.prompt = prompt + self.cont_prompt = continuation_prompt + + # Use expect_exact if we can as it should be faster + self.expects = [prompt] + if re.escape(prompt) == prompt and hasattr(self.child, "expect_exact"): + self._expectf = self.child.expect_exact + else: + self._expectf = self.child.expect + if continuation_prompt: + self.expects.append(continuation_prompt) + if re.escape(continuation_prompt) != continuation_prompt: + self._expectf = self.child.expect + + if extra_init_cmd: + self.expect_prompt() + self.child.sendline(extra_init_cmd) + self.expect_prompt() + + def expect_prompt(self, timeout=-1): + return self._expectf(self.expects, timeout=timeout) + + def run_command(self, command, timeout=-1): + """Pexpect REPLWrapper compatible run_command. + + This will split `command` into lines and feed each one to the shell. + + Args: + command: string of commands separated by newlines, a trailing + newline will cause and empty line to be sent. + timeout: pexpect timeout value. + """ + lines = command.splitlines() + if command[-1] == "\n": + lines.append("") + output = "" + index = 0 + for line in lines: + self.child.sendline(line) + index = self.expect_prompt(timeout=timeout) + output += self.child.before + + if index: + if hasattr(self.child, "kill"): + self.child.kill(signal.SIGINT) + else: + self.child.send("\x03") + self.expect_prompt(timeout=30 if self.child.timeout is None else -1) + raise ValueError("Continuation prompt found at end of commands") + + if self.escape: + output = self.escape.sub("", output) + + return output + + def cmd_nostatus(self, cmd, timeout=-1): + r"""Execute a shell command. + + Returns: + (strip/cleaned \r) output + """ + output = self.run_command(cmd, timeout) + output = output.replace("\r\n", "\n") + if self.echo: + # remove the command + idx = output.find(cmd) + if idx == -1: + logging.warning( + "Didn't find command ('%s') in expected output ('%s')", + cmd, + output, + ) + else: + # Remove up to and including the command from the output stream + output = output[idx + len(cmd) :] + + return output.replace("\r", "").strip() + + def cmd_status(self, cmd, timeout=-1): + r"""Execute a shell command. + + Returns: + status and (strip/cleaned \r) output + """ + # Run the command getting the output + output = self.cmd_nostatus(cmd, timeout) + + # Now get the status + scmd = "echo $?" + rcstr = self.run_command(scmd) + rcstr = rcstr.replace("\r\n", "\n") + if self.echo: + # remove the command + idx = rcstr.find(scmd) + if idx == -1: + if self.echo: + logging.warning( + "Didn't find status ('%s') in expected output ('%s')", + scmd, + rcstr, + ) + try: + rc = int(rcstr) + except Exception: + rc = 255 + else: + rcstr = rcstr[idx + len(scmd) :].strip() + try: + rc = int(rcstr) + except ValueError as error: + logging.error( + "%s: error with expected status output: %s: %s", + self, + error, + rcstr, + exc_info=True, + ) + rc = 255 + return rc, output + + def cmd_raises(self, cmd, timeout=-1): + r"""Execute a shell command. + + Returns: + (strip/cleaned \r) ouptut + + Raises: + CalledProcessError: on non-zero exit status + """ + rc, output = self.cmd_status(cmd, timeout) + if rc: + raise CalledProcessError(rc, cmd, output) + return output + + +# --------------------------- +# Root level utility function +# --------------------------- + + +def get_exec_path(binary): + return commander.get_exec_path(binary) + + +def get_exec_path_host(binary): + return commander.get_exec_path(binary) + + +def get_our_script_path(script): + # would be nice to find this w/o using a path lookup + sdir = os.path.dirname(os.path.abspath(__file__)) + spath = os.path.join(sdir, script) + if os.path.exists(spath): + return spath + return get_exec_path(script) + + +commander = Commander("munet") +roothost = None diff --git a/tests/topotests/munet/cleanup.py b/tests/topotests/munet/cleanup.py new file mode 100644 index 0000000..c641cda --- /dev/null +++ b/tests/topotests/munet/cleanup.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# September 30 2021, Christian Hopps <chopps@labn.net> +# +# Copyright 2021, LabN Consulting, L.L.C. +# +"""Provides functionality to cleanup processes on posix systems.""" +import glob +import logging +import os +import signal + + +def get_pids_with_env(has_var, has_val=None): + result = {} + for pidenv in glob.iglob("/proc/*/environ"): + pid = pidenv.split("/")[2] + try: + with open(pidenv, "rb") as rfb: + envlist = [ + x.decode("utf-8").split("=", 1) for x in rfb.read().split(b"\0") + ] + envlist = [[x[0], ""] if len(x) == 1 else x for x in envlist] + envdict = dict(envlist) + if has_var not in envdict: + continue + if has_val is None: + result[pid] = envdict + elif envdict[has_var] == str(has_val): + result[pid] = envdict + except Exception: + # E.g., process exited and files are gone + pass + return result + + +def _kill_piddict(pids_by_upid, sig): + ourpid = str(os.getpid()) + for upid, pids in pids_by_upid: + logging.info("Sending %s to (%s) of munet pid %s", sig, ", ".join(pids), upid) + for pid in pids: + try: + if pid != ourpid: + cmdline = open(f"/proc/{pid}/cmdline", "r", encoding="ascii").read() + cmdline = cmdline.replace("\x00", " ") + logging.info("killing proc %s (%s)", pid, cmdline) + os.kill(int(pid), sig) + except Exception: + pass + + +def _get_our_pids(): + ourpid = str(os.getpid()) + piddict = get_pids_with_env("MUNET_PID", ourpid) + pids = [x for x in piddict if x != ourpid] + if pids: + return {ourpid: pids} + return {} + + +def _get_other_pids(): + piddict = get_pids_with_env("MUNET_PID") + unet_pids = {d["MUNET_PID"] for d in piddict.values()} + pids_by_upid = {p: set() for p in unet_pids} + for pid, envdict in piddict.items(): + unet_pid = envdict["MUNET_PID"] + pids_by_upid[unet_pid].add(pid) + # Filter out any child pid sets whos munet pid is still running + return {x: y for x, y in pids_by_upid.items() if x not in y} + + +def _get_pids_by_upid(ours): + if ours: + return _get_our_pids() + return _get_other_pids() + + +def _cleanup_pids(ours): + pids_by_upid = _get_pids_by_upid(ours).items() + if not pids_by_upid: + return + + t = "current" if ours else "previous" + logging.info("Reaping %s munet processes", t) + + # _kill_piddict(pids_by_upid, signal.SIGTERM) + + # # Give them 5 second to exit cleanly + # logging.info("Waiting up to 5s to allow for clean exit of abandon'd pids") + # for _ in range(0, 5): + # pids_by_upid = _get_pids_by_upid(ours).items() + # if not pids_by_upid: + # return + # time.sleep(1) + + pids_by_upid = _get_pids_by_upid(ours).items() + _kill_piddict(pids_by_upid, signal.SIGKILL) + + +def cleanup_current(): + """Attempt to cleanup preview runs. + + Currently this only scans for old processes. + """ + _cleanup_pids(True) + + +def cleanup_previous(): + """Attempt to cleanup preview runs. + + Currently this only scans for old processes. + """ + _cleanup_pids(False) diff --git a/tests/topotests/munet/cli.py b/tests/topotests/munet/cli.py new file mode 100644 index 0000000..f631073 --- /dev/null +++ b/tests/topotests/munet/cli.py @@ -0,0 +1,962 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# July 24 2021, Christian Hopps <chopps@labn.net> +# +# Copyright 2021, LabN Consulting, L.L.C. +# +"""A module that implements a CLI.""" +import argparse +import asyncio +import functools +import logging +import multiprocessing +import os +import pty +import re +import readline +import select +import shlex +import socket +import subprocess +import sys +import tempfile +import termios +import tty + + +try: + from . import linux + from .config import list_to_dict_with_key +except ImportError: + # We cannot use relative imports and still run this module directly as a script, and + # there are some use cases where we want to run this file as a script. + sys.path.append(os.path.dirname(os.path.realpath(__file__))) + import linux + + from config import list_to_dict_with_key + + +ENDMARKER = b"\x00END\x00" + +logger = logging.getLogger(__name__) + + +def lineiter(sock): + s = "" + while True: + sb = sock.recv(256) + if not sb: + return + + s += sb.decode("utf-8") + i = s.find("\n") + if i != -1: + yield s[:i] + s = s[i + 1 :] + + +# Would be nice to convert to async, but really not needed as used +def spawn(unet, host, cmd, iow, ns_only): + if sys.stdin.isatty(): + old_tty = termios.tcgetattr(sys.stdin) + tty.setraw(sys.stdin.fileno()) + + try: + master_fd, slave_fd = pty.openpty() + + ns = unet.hosts[host] if host and host != unet else unet + popenf = ns.popen_nsonly if ns_only else ns.popen + + # use os.setsid() make it run in a new process group, or bash job + # control will not be enabled + p = popenf( + cmd, + # _common_prologue, later in call chain, only does this for use_pty == False + preexec_fn=os.setsid, + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + universal_newlines=True, + use_pty=True, + # XXX this is actually implementing "run on host" for real + # skip_pre_cmd=ns_only, + ) + iow.write("\r") + iow.flush() + + while p.poll() is None: + r, _, _ = select.select([sys.stdin, master_fd], [], [], 0.25) + if sys.stdin in r: + d = os.read(sys.stdin.fileno(), 10240) + os.write(master_fd, d) + elif master_fd in r: + o = os.read(master_fd, 10240) + if o: + iow.write(o.decode("utf-8", "ignore")) + iow.flush() + finally: + # restore tty settings back + if sys.stdin.isatty(): + termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_tty) + + +def is_host_regex(restr): + return len(restr) > 2 and restr[0] == "/" and restr[-1] == "/" + + +def get_host_regex(restr): + if len(restr) < 3 or restr[0] != "/" or restr[-1] != "/": + return None + return re.compile(restr[1:-1]) + + +def host_in(restr, names): + """Determine if matcher is a regex that matches one of names.""" + if not (regexp := get_host_regex(restr)): + return restr in names + for name in names: + if regexp.fullmatch(name): + return True + return False + + +def expand_host(restr, names): + """Expand name or regexp into list of hosts.""" + hosts = [] + regexp = get_host_regex(restr) + if not regexp: + assert restr in names + hosts.append(restr) + else: + for name in names: + if regexp.fullmatch(name): + hosts.append(name) + return sorted(hosts) + + +def expand_hosts(restrs, names): + """Expand list of host names or regex into list of hosts.""" + hosts = [] + for restr in restrs: + hosts += expand_host(restr, names) + return sorted(hosts) + + +def host_cmd_split(unet, line, toplevel): + all_hosts = set(unet.hosts) + csplit = line.split() + i = 0 + banner = False + for i, e in enumerate(csplit): + if is_re := is_host_regex(e): + banner = True + if not host_in(e, all_hosts): + if not is_re: + break + else: + i += 1 + + if i == 0 and csplit and csplit[0] == "*": + hosts = sorted(all_hosts) + csplit = csplit[1:] + banner = True + elif i == 0 and csplit and csplit[0] == ".": + hosts = [unet] + csplit = csplit[1:] + else: + hosts = expand_hosts(csplit[:i], all_hosts) + csplit = csplit[i:] + + if not hosts and not csplit[:i]: + if toplevel: + hosts = [unet] + else: + hosts = sorted(all_hosts) + banner = True + + if not csplit: + return hosts, "", "", True + + i = line.index(csplit[0]) + i += len(csplit[0]) + return hosts, csplit[0], line[i:].strip(), banner + + +def win_cmd_host_split(unet, cmd, kinds, defall): + if kinds: + all_hosts = { + x for x in unet.hosts if unet.hosts[x].config.get("kind", "") in kinds + } + else: + all_hosts = set(unet.hosts) + + csplit = cmd.split() + i = 0 + for i, e in enumerate(csplit): + if not host_in(e, all_hosts): + if not is_host_regex(e): + break + else: + i += 1 + + if i == 0 and csplit and csplit[0] == "*": + hosts = sorted(all_hosts) + csplit = csplit[1:] + elif i == 0 and csplit and csplit[0] == ".": + hosts = [unet] + csplit = csplit[1:] + else: + hosts = expand_hosts(csplit[:i], all_hosts) + + if not hosts and defall and not csplit[:i]: + hosts = sorted(all_hosts) + + # Filter hosts based on cmd + cmd = " ".join(csplit[i:]) + return hosts, cmd + + +def proc_readline(fd, prompt, histfile): + """Read a line of input from user while running in a sub-process.""" + # How do we change the command though, that's what's displayed in ps normally + linux.set_process_name("Munet CLI") + try: + # For some reason sys.stdin is fileno == 16 and useless + sys.stdin = os.fdopen(0) + histfile = init_history(None, histfile) + line = input(prompt) + readline.write_history_file(histfile) + if line is None: + os.write(fd, b"\n") + os.write(fd, bytes(f":{str(line)}\n", encoding="utf-8")) + except EOFError: + os.write(fd, b"\n") + except KeyboardInterrupt: + os.write(fd, b"I\n") + except Exception as error: + os.write(fd, bytes(f"E{str(error)}\n", encoding="utf-8")) + + +async def async_input_reader(rfd): + """Read a line of input from the user input sub-process pipe.""" + rpipe = os.fdopen(rfd, mode="r") + reader = asyncio.StreamReader() + + def protocol_factory(): + return asyncio.StreamReaderProtocol(reader) + + loop = asyncio.get_event_loop() + transport, _ = await loop.connect_read_pipe(protocol_factory, rpipe) + o = await reader.readline() + transport.close() + + o = o.decode("utf-8").strip() + if not o: + return None + if o[0] == "I": + raise KeyboardInterrupt() + if o[0] == "E": + raise Exception(o[1:]) + assert o[0] == ":" + return o[1:] + + +# +# A lot of work to add async `input` handling without creating a thread. We cannot use +# threads when unshare_inline is used with pid namespace per kernel clone(2) +# restriction. +# +async def async_input(prompt, histfile): + """Asynchronously read a line from the user.""" + rfd, wfd = os.pipe() + p = multiprocessing.Process(target=proc_readline, args=(wfd, prompt, histfile)) + p.start() + logging.debug("started async_input input process: %s", p) + try: + return await async_input_reader(rfd) + finally: + logging.debug("joining async_input input process") + p.join() + + +def make_help_str(unet): + w = sorted([x if x else "" for x in unet.cli_in_window_cmds]) + ww = unet.cli_in_window_cmds + u = sorted([x if x else "" for x in unet.cli_run_cmds]) + uu = unet.cli_run_cmds + + s = ( + """ +Basic Commands: + cli :: open a secondary CLI window + help :: this help + hosts :: list hosts + quit :: quit the cli + + HOST can be a host or one of the following: + - '*' for all hosts + - '.' for the parent munet + - a regex specified between '/' (e.g., '/rtr.*/') + +New Window Commands:\n""" + + "\n".join([f" {ww[v][0]}\t:: {ww[v][1]}" for v in w]) + + """\nInline Commands:\n""" + + "\n".join([f" {uu[v][0]}\t:: {uu[v][1]}" for v in u]) + + "\n" + ) + return s + + +def get_shcmd(unet, host, kinds, execfmt, ucmd): + if host is None: + h = None + kind = None + elif host is unet or host == "": + h = unet + kind = "" + else: + h = unet.hosts[host] + kind = h.config.get("kind", "") + if kinds and kind not in kinds: + return "" + if not isinstance(execfmt, str): + execfmt = execfmt.get(kind, {}).get("exec", "") + if not execfmt: + return "" + + # Do substitutions for {} in string + numfmt = len(re.findall(r"{\d*}", execfmt)) + if numfmt > 1: + ucmd = execfmt.format(*shlex.split(ucmd)) + elif numfmt: + ucmd = execfmt.format(ucmd) + elif len(re.findall(r"{[a-zA-Z_][0-9a-zA-Z_\.]*}", execfmt)): + if execfmt.endswith('"'): + fstring = "f'''" + execfmt + "'''" + else: + fstring = 'f"""' + execfmt + '"""' + ucmd = eval( # pylint: disable=W0123 + fstring, + globals(), + {"host": h, "unet": unet, "user_input": ucmd}, + ) + else: + # No variable or usercmd substitution at all. + ucmd = execfmt + + # Do substitution for munet variables + ucmd = ucmd.replace("%CONFIGDIR%", str(unet.config_dirname)) + if host is None or host is unet: + ucmd = ucmd.replace("%RUNDIR%", str(unet.rundir)) + return ucmd.replace("%NAME%", ".") + ucmd = ucmd.replace("%RUNDIR%", str(os.path.join(unet.rundir, host))) + if h.mgmt_ip: + ucmd = ucmd.replace("%IPADDR%", str(h.mgmt_ip)) + elif h.mgmt_ip6: + ucmd = ucmd.replace("%IPADDR%", str(h.mgmt_ip6)) + if h.mgmt_ip6: + ucmd = ucmd.replace("%IP6ADDR%", str(h.mgmt_ip6)) + return ucmd.replace("%NAME%", str(host)) + + +async def run_command( + unet, + outf, + line, + execfmt, + banner, + hosts, + toplevel, + kinds, + ns_only=False, + interactive=False, +): + """Runs a command on a set of hosts. + + Runs `execfmt`. Prior to executing the string the following transformations are + performed on it. + + `execfmt` may also be a dictionary of dicitonaries keyed on kind with `exec` holding + the kind's execfmt string. + + - if `{}` is present then `str.format` is called to replace `{}` with any extra + input values after the command and hosts are removed from the input. + - else if any `{digits}` are present then `str.format` is called to replace + `{digits}` with positional args obtained from the addittional user input + first passed to `shlex.split`. + - else f-string style interpolation is performed on the string with + the local variables `host` (the current node object or None), + `unet` (the Munet object), and `user_input` (the additional command input) + defined. + + The output is sent to `outf`. If `ns_only` is True then the `execfmt` is + run using `Commander.cmd_status_nsonly` otherwise it is run with + `Commander.cmd_status`. + """ + if kinds: + logging.info("Filtering hosts to kinds: %s", kinds) + hosts = [x for x in hosts if unet.hosts[x].config.get("kind", "") in kinds] + logging.info("Filtered hosts: %s", hosts) + + if not hosts: + if not toplevel: + return + hosts = [unet] + + # if unknowns := [x for x in hosts if x not in unet.hosts]: + # outf.write("%% Unknown host[s]: %s\n" % ", ".join(unknowns)) + # return + + # if sys.stdin.isatty() and interactive: + if interactive: + for host in hosts: + shcmd = get_shcmd(unet, host, kinds, execfmt, line) + if not shcmd: + continue + if len(hosts) > 1 or banner: + outf.write(f"------ Host: {host} ------\n") + spawn(unet, host if not toplevel else unet, shcmd, outf, ns_only) + if len(hosts) > 1 or banner: + outf.write(f"------- End: {host} ------\n") + outf.write("\n") + return + + aws = [] + for host in hosts: + shcmd = get_shcmd(unet, host, kinds, execfmt, line) + if not shcmd: + continue + if toplevel: + ns = unet + else: + ns = unet.hosts[host] if host and host != unet else unet + if ns_only: + cmdf = ns.async_cmd_status_nsonly + else: + cmdf = ns.async_cmd_status + aws.append(cmdf(shcmd, warn=False, stderr=subprocess.STDOUT)) + + results = await asyncio.gather(*aws, return_exceptions=True) + for host, result in zip(hosts, results): + if isinstance(result, Exception): + o = str(result) + "\n" + rc = -1 + else: + rc, o, _ = result + if len(hosts) > 1 or banner: + outf.write(f"------ Host: {host} ------\n") + if rc: + outf.write(f"*** non-zero exit status: {rc}\n") + outf.write(o) + if len(hosts) > 1 or banner: + outf.write(f"------- End: {host} ------\n") + + +cli_builtins = ["cli", "help", "hosts", "quit"] + + +class Completer: + """A completer class for the CLI.""" + + def __init__(self, unet): + self.unet = unet + + def complete(self, text, state): + line = readline.get_line_buffer() + tokens = line.split() + # print(f"\nXXX: tokens: {tokens} text: '{text}' state: {state}'\n") + + first_token = not tokens or (text and len(tokens) == 1) + + # If we have already have a builtin command we are done + if tokens and tokens[0] in cli_builtins: + return [None] + + cli_run_cmds = set(self.unet.cli_run_cmds.keys()) + top_run_cmds = {x for x in cli_run_cmds if self.unet.cli_run_cmds[x][3]} + cli_run_cmds -= top_run_cmds + cli_win_cmds = set(self.unet.cli_in_window_cmds.keys()) + hosts = set(self.unet.hosts.keys()) + is_window_cmd = bool(tokens) and tokens[0] in cli_win_cmds + done_set = set() + if bool(tokens): + if text: + done_set = set(tokens[:-1]) + else: + done_set = set(tokens) + + # Determine the domain for completions + if not tokens or first_token: + all_cmds = ( + set(cli_builtins) | hosts | cli_run_cmds | cli_win_cmds | top_run_cmds + ) + elif is_window_cmd: + all_cmds = hosts + elif tokens and tokens[0] in top_run_cmds: + # nothing to complete if a top level command + pass + elif not bool(done_set & cli_run_cmds): + all_cmds = hosts | cli_run_cmds + + if not text: + completes = all_cmds + else: + # print(f"\nXXX: all_cmds: {all_cmds} text: '{text}'\n") + completes = {x + " " for x in all_cmds if x.startswith(text)} + + # print(f"\nXXX: completes: {completes} text: '{text}' state: {state}'\n") + # remove any completions already present + completes -= done_set + completes = sorted(completes) + [None] + return completes[state] + + +async def doline( + unet, line, outf, background=False, notty=False +): # pylint: disable=R0911 + line = line.strip() + m = re.fullmatch(r"^(\S+)(?:\s+(.*))?$", line) + if not m: + return True + + cmd = m.group(1) + nline = m.group(2) if m.group(2) else "" + + if cmd in ("q", "quit"): + return False + + if cmd == "help": + outf.write(make_help_str(unet)) + return True + if cmd in ("h", "hosts"): + outf.write(f"% Hosts:\t{' '.join(sorted(unet.hosts.keys()))}\n") + return True + if cmd == "cli": + await remote_cli( + unet, + "secondary> ", + "Secondary CLI", + background, + ) + return True + + # + # In window commands + # + + if cmd in unet.cli_in_window_cmds: + execfmt, toplevel, kinds, kwargs = unet.cli_in_window_cmds[cmd][2:] + + # if toplevel: + # ucmd = " ".join(nline.split()) + # else: + hosts, ucmd = win_cmd_host_split(unet, nline, kinds, False) + if not hosts: + if not toplevel: + return True + hosts = [unet] + + if isinstance(execfmt, str): + found_brace = "{}" in execfmt + else: + found_brace = False + for d in execfmt.values(): + if "{}" in d["exec"]: + found_brace = True + break + if not found_brace and ucmd and not toplevel: + # CLI command does not expect user command so treat as hosts of which some + # must be unknown + unknowns = [x for x in ucmd.split() if x not in unet.hosts] + outf.write(f"% Unknown host[s]: {' '.join(unknowns)}\n") + return True + + try: + if not hosts and toplevel: + hosts = [unet] + + for host in hosts: + shcmd = get_shcmd(unet, host, kinds, execfmt, ucmd) + if toplevel or host == unet: + unet.run_in_window(shcmd, **kwargs) + else: + unet.hosts[host].run_in_window(shcmd, **kwargs) + except Exception as error: + outf.write(f"% Error: {error}\n") + return True + + # + # Inline commands + # + + toplevel = unet.cli_run_cmds[cmd][3] if cmd in unet.cli_run_cmds else False + # if toplevel: + # logging.debug("top-level: cmd: '%s' nline: '%s'", cmd, nline) + # hosts = None + # banner = False + # else: + + hosts, cmd, nline, banner = host_cmd_split(unet, line, toplevel) + hoststr = "munet" if hosts == [unet] else f"{hosts}" + logging.debug("hosts: '%s' cmd: '%s' nline: '%s'", hoststr, cmd, nline) + + if cmd in unet.cli_run_cmds: + pass + elif "" in unet.cli_run_cmds: + nline = f"{cmd} {nline}" + cmd = "" + else: + outf.write(f"% Unknown command: {cmd} {nline}\n") + return True + + execfmt, toplevel, kinds, ns_only, interactive = unet.cli_run_cmds[cmd][2:] + if interactive and notty: + outf.write("% Error: interactive command must be run from primary CLI\n") + return True + + await run_command( + unet, + outf, + nline, + execfmt, + banner, + hosts, + toplevel, + kinds, + ns_only, + interactive, + ) + + return True + + +async def cli_client(sockpath, prompt="munet> "): + """Implement the user-facing CLI for a remote munet reached by a socket.""" + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.settimeout(10) + sock.connect(sockpath) + + # Go into full non-blocking mode now + sock.settimeout(None) + + print("\n--- Munet CLI Starting ---\n\n") + while True: + line = input(prompt) + if line is None: + return + + # Need to put \n back + line += "\n" + + # Send the CLI command + sock.send(line.encode("utf-8")) + + def bendswith(b, sentinel): + slen = len(sentinel) + return len(b) >= slen and b[-slen:] == sentinel + + # Collect the output + rb = b"" + while not bendswith(rb, ENDMARKER): + lb = sock.recv(4096) + if not lb: + return + rb += lb + + # Remove the marker + rb = rb[: -len(ENDMARKER)] + + # Write the output + sys.stdout.write(rb.decode("utf-8", "ignore")) + + +async def local_cli(unet, outf, prompt, histfile, background): + """Implement the user-side CLI for local munet.""" + assert unet is not None + completer = Completer(unet) + readline.parse_and_bind("tab: complete") + readline.set_completer(completer.complete) + + print("\n--- Munet CLI Starting ---\n\n") + while True: + try: + line = await async_input(prompt, histfile) + if line is None: + return + + if not await doline(unet, line, outf, background): + return + except KeyboardInterrupt: + outf.write("%% Caught KeyboardInterrupt\nUse ^D or 'quit' to exit") + + +def init_history(unet, histfile): + try: + if histfile is None: + histfile = os.path.expanduser("~/.munet-history.txt") + if not os.path.exists(histfile): + if unet: + unet.cmd("touch " + histfile) + else: + subprocess.run("touch " + histfile, shell=True, check=True) + if histfile: + readline.read_history_file(histfile) + return histfile + except Exception as error: + logging.warning("init_history failed: %s", error) + return None + + +async def cli_client_connected(unet, background, reader, writer): + """Handle CLI commands inside the munet process from a socket.""" + # # Go into full non-blocking mode now + # client.settimeout(None) + logging.debug("cli client connected") + while True: + line = await reader.readline() + if not line: + logging.debug("client closed cli connection") + break + line = line.decode("utf-8").strip() + + class EncodingFile: + """Wrap a writer to encode in utf-8.""" + + def __init__(self, writer): + self.writer = writer + + def write(self, x): + self.writer.write(x.encode("utf-8", "ignore")) + + def flush(self): + self.writer.flush() + + if not await doline(unet, line, EncodingFile(writer), background, notty=True): + logging.debug("server closing cli connection") + return + + writer.write(ENDMARKER) + await writer.drain() + + +async def remote_cli(unet, prompt, title, background): + """Open a CLI in a new window.""" + try: + if not unet.cli_sockpath: + sockpath = os.path.join(tempfile.mkdtemp("-sockdir", "pty-"), "cli.sock") + ccfunc = functools.partial(cli_client_connected, unet, background) + s = await asyncio.start_unix_server(ccfunc, path=sockpath) + unet.cli_server = asyncio.create_task(s.serve_forever(), name="cli-task") + unet.cli_sockpath = sockpath + logging.info("server created on :\n%s\n", sockpath) + + # Open a new window with a new CLI + python_path = await unet.async_get_exec_path(["python3", "python"]) + us = os.path.realpath(__file__) + cmd = f"{python_path} {us}" + if unet.cli_histfile: + cmd += " --histfile=" + unet.cli_histfile + if prompt: + cmd += f" --prompt='{prompt}'" + cmd += " " + unet.cli_sockpath + unet.run_in_window(cmd, title=title, background=False) + except Exception as error: + logging.error("cli server: unexpected exception: %s", error) + + +def add_cli_in_window_cmd( + unet, name, helpfmt, helptxt, execfmt, toplevel, kinds, **kwargs +): + """Adds a CLI command to the CLI. + + The command `cmd` is added to the commands executable by the user from the CLI. See + `base.Commander.run_in_window` for the arguments that can be passed in `args` and + `kwargs` to this function. + + Args: + unet: unet object + name: command string (no spaces) + helpfmt: format of command to display in help (left side) + helptxt: help string for command (right side) + execfmt: interpreter `cmd` to pass to `host.run_in_window()`, if {} present then + allow for user commands to be entered and inserted. May also be a dict of dict + keyed on kind with sub-key of "exec" providing the `execfmt` string for that + kind. + toplevel: run command in common top-level namespaec not inside hosts + kinds: limit CLI command to nodes which match list of kinds. + **kwargs: keyword args to pass to `host.run_in_window()` + """ + unet.cli_in_window_cmds[name] = (helpfmt, helptxt, execfmt, toplevel, kinds, kwargs) + + +def add_cli_run_cmd( + unet, + name, + helpfmt, + helptxt, + execfmt, + toplevel, + kinds, + ns_only=False, + interactive=False, +): + """Adds a CLI command to the CLI. + + The command `cmd` is added to the commands executable by the user from the CLI. + See `run_command` above in the `doline` function and for the arguments that can + be passed in to this function. + + Args: + unet: unet object + name: command string (no spaces) + helpfmt: format of command to display in help (left side) + helptxt: help string for command (right side) + execfmt: format string to insert user cmds into for execution. May also be a + dict of dict keyed on kind with sub-key of "exec" providing the `execfmt` + string for that kind. + toplevel: run command in common top-level namespaec not inside hosts + kinds: limit CLI command to nodes which match list of kinds. + ns_only: Should execute the command on the host vs in the node namespace. + interactive: Should execute the command inside an allocated pty (interactive) + """ + unet.cli_run_cmds[name] = ( + helpfmt, + helptxt, + execfmt, + toplevel, + kinds, + ns_only, + interactive, + ) + + +def add_cli_config(unet, config): + """Adds CLI commands based on config. + + All exec strings will have %CONFIGDIR%, %NAME% and %RUNDIR% replaced with the + corresponding config directory and the current nodes `name` and `rundir`. + Additionally, the exec string will have f-string style interpolation performed + with the local variables `host` (node object or None), `unet` (Munet object) and + `user_input` (if provided to the CLI command) defined. + + The format of the config dictionary can be seen in the following example. + The first list entry represents the default command because it has no `name` key. + + commands: + - help: "run the given FRR command using vtysh" + format: "[HOST ...] FRR-CLI-COMMAND" + exec: "vtysh -c {}" + ns-only: false # the default + interactive: false # the default + - name: "vtysh" + help: "Open a FRR CLI inside new terminal[s] on the given HOST[s]" + format: "vtysh HOST [HOST ...]" + exec: "vtysh" + new-window: true + - name: "capture" + help: "Capture packets on a given network" + format: "pcap NETWORK" + exec: "tshark -s 9200 -i {0} -w /tmp/capture-{0}.pcap" + new-window: true + top-level: true # run in top-level container namespace, above hosts + + The `new_window` key can also be a dictionary which will be passed as keyward + arguments to the `Commander.run_in_window()` function. + + Args: + unet: unet object + config: dictionary of cli config + """ + for cli_cmd in config.get("commands", []): + name = cli_cmd.get("name", None) + helpfmt = cli_cmd.get("format", "") + helptxt = cli_cmd.get("help", "") + execfmt = list_to_dict_with_key(cli_cmd.get("exec-kind"), "kind") + if not execfmt: + execfmt = cli_cmd.get("exec", "bash -c '{}'") + toplevel = cli_cmd.get("top-level", False) + kinds = cli_cmd.get("kinds", []) + stdargs = (unet, name, helpfmt, helptxt, execfmt, toplevel, kinds) + new_window = cli_cmd.get("new-window", None) + if isinstance(new_window, dict): + add_cli_in_window_cmd(*stdargs, **new_window) + elif bool(new_window): + add_cli_in_window_cmd(*stdargs) + else: + # on-host is deprecated it really implemented "ns-only" + add_cli_run_cmd( + *stdargs, + cli_cmd.get("ns-only", cli_cmd.get("on-host")), + cli_cmd.get("interactive", False), + ) + + +def cli( + unet, + histfile=None, + sockpath=None, + force_window=False, + title=None, + prompt=None, + background=True, +): + asyncio.run( + async_cli(unet, histfile, sockpath, force_window, title, prompt, background) + ) + + +async def async_cli( + unet, + histfile=None, + sockpath=None, + force_window=False, + title=None, + prompt=None, + background=True, +): + if prompt is None: + prompt = "munet> " + + if force_window or not sys.stdin.isatty(): + await remote_cli(unet, prompt, title, background) + + if not unet: + logger.debug("client-cli using sockpath %s", sockpath) + + try: + if sockpath: + await cli_client(sockpath, prompt) + else: + await local_cli(unet, sys.stdout, prompt, histfile, background) + except KeyboardInterrupt: + print("\n...^C exiting CLI") + except EOFError: + pass + except Exception as ex: + logger.critical("cli: got exception: %s", ex, exc_info=True) + raise + + +if __name__ == "__main__": + # logging.basicConfig(level=logging.DEBUG, filename="/tmp/topotests/cli-client.log") + logging.basicConfig(level=logging.DEBUG) + logger = logging.getLogger("cli-client") + logger.info("Start logging cli-client") + + parser = argparse.ArgumentParser() + parser.add_argument("--histfile", help="file to user for history") + parser.add_argument("--prompt", help="prompt string to use") + parser.add_argument("socket", help="path to pair of sockets to communicate over") + cli_args = parser.parse_args() + + cli_prompt = cli_args.prompt if cli_args.prompt else "munet> " + asyncio.run( + async_cli( + None, + cli_args.histfile, + cli_args.socket, + prompt=cli_prompt, + background=False, + ) + ) diff --git a/tests/topotests/munet/compat.py b/tests/topotests/munet/compat.py new file mode 100644 index 0000000..e82a7d5 --- /dev/null +++ b/tests/topotests/munet/compat.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# November 16 2022, Christian Hopps <chopps@labn.net> +# +# Copyright (c) 2022, LabN Consulting, L.L.C. +# +"""Provide compatible APIs.""" + + +class PytestConfig: + """Pytest config duck-type-compatible object using argprase args.""" + + class Namespace: + """A namespace defined by a dictionary of values.""" + + def __init__(self, args): + self.args = args + + def __getattr__(self, attr): + return self.args[attr] if attr in self.args else None + + def __init__(self, args): + self.args = vars(args) + self.option = PytestConfig.Namespace(self.args) + + def getoption(self, name, default=None, skip=False): + assert not skip + if name.startswith("--"): + name = name[2:] + name = name.replace("-", "_") + if name in self.args: + return self.args[name] if self.args[name] is not None else default + return default diff --git a/tests/topotests/munet/config.py b/tests/topotests/munet/config.py new file mode 100644 index 0000000..2870ae6 --- /dev/null +++ b/tests/topotests/munet/config.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# June 25 2022, Christian Hopps <chopps@gmail.com> +# +# Copyright (c) 2021-2022, LabN Consulting, L.L.C. +# +"""A module that defines common configuration utility functions.""" +import logging + +from collections.abc import Iterable +from copy import deepcopy +from typing import overload + + +def find_with_kv(lst, k, v): + if lst: + for e in lst: + if k in e and e[k] == v: + return e + return {} + + +def find_all_with_kv(lst, k, v): + rv = [] + if lst: + for e in lst: + if k in e and e[k] == v: + rv.append(e) + return rv + + +def find_matching_net_config(name, cconf, oconf): + p = find_all_with_kv(oconf.get("connections", {}), "to", name) + if not p: + return {} + + rname = cconf.get("remote-name", None) + if not rname: + return p[0] + + return find_with_kv(p, "name", rname) + + +def merge_using_key(a, b, k): + # First get a dict of indexes in `a` for the key value of `k` in objects of `a` + m = list(a) + mi = {o[k]: i for i, o in enumerate(m)} + for o in b: + bkv = o[k] + if bkv in mi: + m[mi[bkv]] = o + else: + mi[bkv] = len(m) + m.append(o) + return m + + +def list_to_dict_with_key(lst, k): + """Convert a YANG styl list of objects to dict of objects. + + This function converts a YANG style list of objects (dictionaries) to a plain python + dictionary of objects (dictionaries). The value for the supplied key for each + object is used to store the object in the new diciontary. + + This only works for lists of objects which are keyed on a single contained value. + + Args: + lst: a *list* of python dictionary objects. + k: the key value contained in each dictionary object in the list. + + Returns: + A dictionary of objects (dictionaries). + """ + return {x[k]: x for x in (lst if lst else [])} + + +def config_to_dict_with_key(c, ck, k): + """Convert the config item from a list of objects to dict. + + Use :py:func:`list_to_dict_with_key` to convert the list of objects + at ``c[ck]`` to a dict of the objects using the key ``k``. + + Args: + c: config dictionary + ck: The key identifying the list of objects from ``c``. + k: The key to pass to :py:func:`list_to_dict_with_key`. + + Returns: + A dictionary of objects (dictionaries). + """ + c[ck] = list_to_dict_with_key(c.get(ck, []), k) + return c[ck] + + +@overload +def config_subst(config: str, **kwargs) -> str: + ... + + +@overload +def config_subst(config: Iterable, **kwargs) -> Iterable: + ... + + +def config_subst(config: Iterable, **kwargs) -> Iterable: + if isinstance(config, str): + if "%RUNDIR%/%NAME%" in config: + config = config.replace("%RUNDIR%/%NAME%", "%RUNDIR%") + logging.warning( + "config '%RUNDIR%/%NAME%' should be changed to '%RUNDIR%' only, " + "converting automatically for now." + ) + for name, value in kwargs.items(): + config = config.replace(f"%{name.upper()}%", str(value)) + elif isinstance(config, Iterable): + try: + return {k: config_subst(config[k], **kwargs) for k in config} + except (KeyError, TypeError): + return [config_subst(x, **kwargs) for x in config] + return config + + +def value_merge_deepcopy(s1, s2): + """Merge values using deepcopy. + + Create a deepcopy of the result of merging the values from dicts ``s1`` and ``s2``. + If a key exists in both ``s1`` and ``s2`` the value from ``s2`` is used." + """ + d = {} + for k, v in s1.items(): + if k in s2: + d[k] = deepcopy(s2[k]) + else: + d[k] = deepcopy(v) + return d + + +def merge_kind_config(kconf, config): + mergekeys = kconf.get("merge", []) + config = deepcopy(config) + new = deepcopy(kconf) + for k in new: + if k not in config: + continue + + if k not in mergekeys: + new[k] = config[k] + elif isinstance(new[k], list): + new[k].extend(config[k]) + elif isinstance(new[k], dict): + new[k] = {**new[k], **config[k]} + else: + new[k] = config[k] + for k in config: + if k not in new: + new[k] = config[k] + return new + + +def cli_opt_list(option_list): + if not option_list: + return [] + if isinstance(option_list, str): + return [x for x in option_list.split(",") if x] + return [x for x in option_list if x] + + +def name_in_cli_opt_str(name, option_list): + ol = cli_opt_list(option_list) + return name in ol or "all" in ol + + +class ConfigOptionsProxy: + """Proxy options object to fill in for any missing pytest config.""" + + class DefNoneObject: + """An object that returns None for any attribute access.""" + + def __getattr__(self, attr): + return None + + def __init__(self, pytestconfig=None): + if isinstance(pytestconfig, ConfigOptionsProxy): + self.config = pytestconfig.config + self.option = self.config.option + else: + self.config = pytestconfig + if self.config: + self.option = self.config.option + else: + self.option = ConfigOptionsProxy.DefNoneObject() + + def getoption(self, opt, default=None): + if not self.config: + return default + + try: + value = self.config.getoption(opt) + return value if value is not None else default + except ValueError: + return default + + def get_option(self, opt, default=None): + return self.getoption(opt, default) + + def get_option_list(self, opt): + value = self.get_option(opt, "") + return cli_opt_list(value) + + def name_in_option_list(self, name, opt): + optlist = self.get_option_list(opt) + return "all" in optlist or name in optlist diff --git a/tests/topotests/munet/kinds.yaml b/tests/topotests/munet/kinds.yaml new file mode 100644 index 0000000..0c278d3 --- /dev/null +++ b/tests/topotests/munet/kinds.yaml @@ -0,0 +1,84 @@ +version: 1 +kinds: + - name: frr + cap-add: + # Zebra requires these + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + - AUDIT_WRITE # needed for ssh pty allocation + - name: ceos + init: false + shell: false + merge: ["env"] + # Should we cap-drop some of these in privileged mode? + # ceos kind is special. munet will add args to /sbin/init for each + # environment variable of the form `systemd.setenv=ENVNAME=VALUE` for each + # environment varialbe named ENVNAME with a value of `VALUE`. If cmd: is + # changed to anything but `/sbin/init` munet will not do this. + cmd: /sbin/init + privileged: true + env: + - name: "EOS_PLATFORM" + value: "ceoslab" + - name: "container" + value: "docker" + - name: "ETBA" + value: "4" + - name: "SKIP_ZEROTOUCH_BARRIER_IN_SYSDBINIT" + value: "1" + - name: "INTFTYPE" + value: "eth" + - name: "MAPETH0" + value: "1" + - name: "MGMT_INTF" + value: "eth0" + - name: "CEOS" + value: "1" + + # cap-add: + # # cEOS requires these, except GNMI still doesn't work + # # - NET_ADMIN + # # - NET_RAW + # # - SYS_ADMIN + # # - SYS_RESOURCE # Required for the CLI + + # All Caps + # - AUDIT_CONTROL + # - AUDIT_READ + # - AUDIT_WRITE + # - BLOCK_SUSPEND + # - CHOWN + # - DAC_OVERRIDE + # - DAC_READ_SEARCH + # - FOWNER + # - FSETID + # - IPC_LOCK + # - IPC_OWNER + # - KILL + # - LEASE + # - LINUX_IMMUTABLE + # - MKNOD + # - NET_ADMIN + # - NET_BIND_SERVICE + # - NET_BROADCAST + # - NET_RAW + # - SETFCAP + # - SETGID + # - SETPCAP + # - SETUID + # - SYSLOG + # - SYS_ADMIN + # - SYS_BOOT + # - SYS_CHROOT + # - SYS_MODULE + # - SYS_NICE + # - SYS_PACCT + # - SYS_PTRACE + # - SYS_RAWIO + # - SYS_RESOURCE + # - SYS_TIME + # - SYS_TTY_CONFIG + # - WAKE_ALARM + # - MAC_ADMIN - Smack project? + # - MAC_OVERRIDE - Smack project? diff --git a/tests/topotests/munet/linux.py b/tests/topotests/munet/linux.py new file mode 100644 index 0000000..417f745 --- /dev/null +++ b/tests/topotests/munet/linux.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# June 10 2022, Christian Hopps <chopps@labn.net> +# +# Copyright (c) 2022, LabN Consulting, L.L.C. +# +"""A module that gives access to linux unshare system call.""" + +import ctypes # pylint: disable=C0415 +import ctypes.util # pylint: disable=C0415 +import errno +import functools +import os + + +libc = None + + +def raise_oserror(enum): + s = errno.errorcode[enum] if enum in errno.errorcode else str(enum) + error = OSError(s) + error.errno = enum + error.strerror = s + raise error + + +def _load_libc(): + global libc # pylint: disable=W0601,W0603 + if libc: + return + lcpath = ctypes.util.find_library("c") + libc = ctypes.CDLL(lcpath, use_errno=True) + + +def pause(): + if not libc: + _load_libc() + libc.pause() + + +MS_RDONLY = 1 +MS_NOSUID = 1 << 1 +MS_NODEV = 1 << 2 +MS_NOEXEC = 1 << 3 +MS_SYNCHRONOUS = 1 << 4 +MS_REMOUNT = 1 << 5 +MS_MANDLOCK = 1 << 6 +MS_DIRSYNC = 1 << 7 +MS_NOSYMFOLLOW = 1 << 8 +MS_NOATIME = 1 << 10 +MS_NODIRATIME = 1 << 11 +MS_BIND = 1 << 12 +MS_MOVE = 1 << 13 +MS_REC = 1 << 14 +MS_SILENT = 1 << 15 +MS_POSIXACL = 1 << 16 +MS_UNBINDABLE = 1 << 17 +MS_PRIVATE = 1 << 18 +MS_SLAVE = 1 << 19 +MS_SHARED = 1 << 20 +MS_RELATIME = 1 << 21 +MS_KERNMOUNT = 1 << 22 +MS_I_VERSION = 1 << 23 +MS_STRICTATIME = 1 << 24 +MS_LAZYTIME = 1 << 25 + + +def mount(source, target, fs, flags=0, options=""): + if not libc: + _load_libc() + libc.mount.argtypes = ( + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_ulong, + ctypes.c_char_p, + ) + fsenc = fs.encode() if fs else None + optenc = options.encode() if options else None + ret = libc.mount(source.encode(), target.encode(), fsenc, flags, optenc) + if ret < 0: + err = ctypes.get_errno() + raise OSError( + err, + f"Error mounting {source} ({fs}) on {target}" + f" with options '{options}': {os.strerror(err)}", + ) + + +# unmout options +MNT_FORCE = 0x1 +MNT_DETACH = 0x2 +MNT_EXPIRE = 0x4 +UMOUNT_NOFOLLOW = 0x8 + + +def umount(target, options): + if not libc: + _load_libc() + libc.umount.argtypes = (ctypes.c_char_p, ctypes.c_uint) + + ret = libc.umount(target.encode(), int(options)) + if ret < 0: + err = ctypes.get_errno() + raise OSError( + err, + f"Error umounting {target} with options '{options}': {os.strerror(err)}", + ) + + +def pidfd_open(pid, flags=0): + if hasattr(os, "pidfd_open") and os.pidfd_open is not pidfd_open: + return os.pidfd_open(pid, flags) # pylint: disable=no-member + + if not libc: + _load_libc() + + try: + pfof = libc.pidfd_open + except AttributeError: + __NR_pidfd_open = 434 + _pidfd_open = libc.syscall + _pidfd_open.restype = ctypes.c_int + _pidfd_open.argtypes = ctypes.c_long, ctypes.c_uint, ctypes.c_uint + pfof = functools.partial(_pidfd_open, __NR_pidfd_open) + + fd = pfof(int(pid), int(flags)) + if fd == -1: + raise_oserror(ctypes.get_errno()) + + return fd + + +if not hasattr(os, "pidfd_open"): + os.pidfd_open = pidfd_open + + +def setns(fd, nstype): # noqa: D402 + """See setns(2) manpage.""" + if not libc: + _load_libc() + + if libc.setns(int(fd), int(nstype)) == -1: + raise_oserror(ctypes.get_errno()) + + +def unshare(flags): # noqa: D402 + """See unshare(2) manpage.""" + if not libc: + _load_libc() + + if libc.unshare(int(flags)) == -1: + raise_oserror(ctypes.get_errno()) + + +CLONE_NEWTIME = 0x00000080 +CLONE_VM = 0x00000100 +CLONE_FS = 0x00000200 +CLONE_FILES = 0x00000400 +CLONE_SIGHAND = 0x00000800 +CLONE_PIDFD = 0x00001000 +CLONE_PTRACE = 0x00002000 +CLONE_VFORK = 0x00004000 +CLONE_PARENT = 0x00008000 +CLONE_THREAD = 0x00010000 +CLONE_NEWNS = 0x00020000 +CLONE_SYSVSEM = 0x00040000 +CLONE_SETTLS = 0x00080000 +CLONE_PARENT_SETTID = 0x00100000 +CLONE_CHILD_CLEARTID = 0x00200000 +CLONE_DETACHED = 0x00400000 +CLONE_UNTRACED = 0x00800000 +CLONE_CHILD_SETTID = 0x01000000 +CLONE_NEWCGROUP = 0x02000000 +CLONE_NEWUTS = 0x04000000 +CLONE_NEWIPC = 0x08000000 +CLONE_NEWUSER = 0x10000000 +CLONE_NEWPID = 0x20000000 +CLONE_NEWNET = 0x40000000 +CLONE_IO = 0x80000000 + +clone_flag_names = { + CLONE_NEWTIME: "CLONE_NEWTIME", + CLONE_VM: "CLONE_VM", + CLONE_FS: "CLONE_FS", + CLONE_FILES: "CLONE_FILES", + CLONE_SIGHAND: "CLONE_SIGHAND", + CLONE_PIDFD: "CLONE_PIDFD", + CLONE_PTRACE: "CLONE_PTRACE", + CLONE_VFORK: "CLONE_VFORK", + CLONE_PARENT: "CLONE_PARENT", + CLONE_THREAD: "CLONE_THREAD", + CLONE_NEWNS: "CLONE_NEWNS", + CLONE_SYSVSEM: "CLONE_SYSVSEM", + CLONE_SETTLS: "CLONE_SETTLS", + CLONE_PARENT_SETTID: "CLONE_PARENT_SETTID", + CLONE_CHILD_CLEARTID: "CLONE_CHILD_CLEARTID", + CLONE_DETACHED: "CLONE_DETACHED", + CLONE_UNTRACED: "CLONE_UNTRACED", + CLONE_CHILD_SETTID: "CLONE_CHILD_SETTID", + CLONE_NEWCGROUP: "CLONE_NEWCGROUP", + CLONE_NEWUTS: "CLONE_NEWUTS", + CLONE_NEWIPC: "CLONE_NEWIPC", + CLONE_NEWUSER: "CLONE_NEWUSER", + CLONE_NEWPID: "CLONE_NEWPID", + CLONE_NEWNET: "CLONE_NEWNET", + CLONE_IO: "CLONE_IO", +} + + +def clone_flag_string(flags): + ns = [v for k, v in clone_flag_names.items() if k & flags] + if ns: + return "|".join(ns) + return "None" + + +namespace_files = { + CLONE_NEWUSER: "ns/user", + CLONE_NEWCGROUP: "ns/cgroup", + CLONE_NEWIPC: "ns/ipc", + CLONE_NEWUTS: "ns/uts", + CLONE_NEWNET: "ns/net", + CLONE_NEWPID: "ns/pid_for_children", + CLONE_NEWNS: "ns/mnt", + CLONE_NEWTIME: "ns/time_for_children", +} + +PR_SET_PDEATHSIG = 1 +PR_GET_PDEATHSIG = 2 +PR_SET_NAME = 15 +PR_GET_NAME = 16 + + +def set_process_name(name): + if not libc: + _load_libc() + + # Why does uncommenting this cause failure? + # libc.prctl.argtypes = ( + # ctypes.c_int, + # ctypes.c_ulong, + # ctypes.c_ulong, + # ctypes.c_ulong, + # ctypes.c_ulong, + # ) + + s = ctypes.create_string_buffer(bytes(name, encoding="ascii")) + sr = ctypes.byref(s) + libc.prctl(PR_SET_NAME, sr, 0, 0, 0) + + +def set_parent_death_signal(signum): + if not libc: + _load_libc() + + # Why does uncommenting this cause failure? + libc.prctl.argtypes = ( + ctypes.c_int, + ctypes.c_ulong, + ctypes.c_ulong, + ctypes.c_ulong, + ctypes.c_ulong, + ) + + libc.prctl(PR_SET_PDEATHSIG, signum, 0, 0, 0) diff --git a/tests/topotests/munet/logconf-mutest.yaml b/tests/topotests/munet/logconf-mutest.yaml new file mode 100644 index 0000000..b450fb9 --- /dev/null +++ b/tests/topotests/munet/logconf-mutest.yaml @@ -0,0 +1,84 @@ +version: 1 +formatters: + brief: + format: '%(levelname)5s: %(message)s' + operfmt: + class: munet.mulog.ColorFormatter + format: ' ------| %(message)s' + exec: + format: '%(asctime)s %(levelname)5s: %(name)s: %(message)s' + output: + format: '%(asctime)s %(levelname)5s: OUTPUT: %(message)s' + results: + # format: '%(asctime)s %(levelname)5s: %(message)s' + format: '%(message)s' + +handlers: + console: + level: WARNING + class: logging.StreamHandler + formatter: brief + stream: ext://sys.stderr + info_console: + level: INFO + class: logging.StreamHandler + formatter: brief + stream: ext://sys.stderr + oper_console: + level: DEBUG + class: logging.StreamHandler + formatter: operfmt + stream: ext://sys.stderr + exec: + level: DEBUG + class: logging.FileHandler + formatter: exec + filename: mutest-exec.log + mode: w + output: + level: DEBUG + class: munet.mulog.MultiFileHandler + root_path: "mutest.output" + formatter: output + filename: mutest-output.log + mode: w + results: + level: INFO + class: munet.mulog.MultiFileHandler + root_path: "mutest.results" + new_handler_level: DEBUG + formatter: results + filename: mutest-results.log + mode: w + +root: + level: DEBUG + handlers: [ "console", "exec" ] + +loggers: + # These are some loggers that get used... + # munet: + # level: DEBUG + # propagate: true + # munet.base.commander + # level: DEBUG + # propagate: true + # mutest.error: + # level: DEBUG + # propagate: true + mutest.output: + level: DEBUG + handlers: ["output", "exec"] + propagate: false + mutest.results: + level: DEBUG + handlers: [ "info_console", "exec", "output", "results" ] + # We don't propagate this b/c we want a lower level accept on the console + # Instead we use info_console and exec to cover what root would log to. + propagate: false + # This is used to debug the operation of mutest + mutest.oper: + # Records are emitted at DEBUG so this will normally filter everything + level: INFO + handlers: [ "oper_console" ] + propagate: false diff --git a/tests/topotests/munet/logconf.yaml b/tests/topotests/munet/logconf.yaml new file mode 100644 index 0000000..430ee20 --- /dev/null +++ b/tests/topotests/munet/logconf.yaml @@ -0,0 +1,32 @@ +version: 1 +formatters: + brief: + format: '%(asctime)s: %(levelname)s: %(message)s' + precise: + format: '%(asctime)s %(levelname)s: %(name)s: %(message)s' + +handlers: + console: + class: logging.StreamHandler + formatter: brief + level: INFO + stream: ext://sys.stderr + file: + class: logging.FileHandler + formatter: precise + level: DEBUG + filename: munet-exec.log + mode: w + +root: + level: DEBUG + handlers: [ "console", "file" ] + +# these are some loggers that get used. +# loggers: +# munet: +# level: DEBUG +# propagate: true +# munet.base.commander +# level: DEBUG +# propagate: true diff --git a/tests/topotests/munet/mucmd.py b/tests/topotests/munet/mucmd.py new file mode 100644 index 0000000..5518c6d --- /dev/null +++ b/tests/topotests/munet/mucmd.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# December 5 2021, Christian Hopps <chopps@labn.net> +# +# Copyright 2021, LabN Consulting, L.L.C. +# +"""A command that allows external command execution inside nodes.""" +import argparse +import json +import os +import subprocess +import sys + +from pathlib import Path + + +def newest_file_in(filename, paths, has_sibling=None): + new = None + newst = None + items = (x for y in paths for x in Path(y).rglob(filename)) + for e in items: + st = os.stat(e) + if has_sibling and not e.parent.joinpath(has_sibling).exists(): + continue + if not new or st.st_mtime_ns > newst.st_mtime_ns: + new = e + newst = st + continue + return new, newst + + +def main(*args): + ap = argparse.ArgumentParser(args) + ap.add_argument("-d", "--rundir", help="runtime directory for tempfiles, logs, etc") + ap.add_argument("node", nargs="?", help="node to enter or run command inside") + ap.add_argument( + "shellcmd", + nargs=argparse.REMAINDER, + help="optional shell-command to execute on NODE", + ) + args = ap.parse_args() + if args.rundir: + configpath = Path(args.rundir).joinpath("config.json") + else: + configpath, _ = newest_file_in( + "config.json", + ["/tmp/munet", "/tmp/mutest", "/tmp/unet-test"], + has_sibling=args.node, + ) + print(f'Using "{configpath}"') + + if not configpath.exists(): + print(f'"{configpath}" not found') + return 1 + rundir = configpath.parent + + nodes = [] + config = json.load(open(configpath, encoding="utf-8")) + nodes = list(config.get("topology", {}).get("nodes", [])) + envcfg = config.get("mucmd", {}).get("env", {}) + + # If args.node is not a node it's part of shellcmd + if args.node and args.node not in nodes: + if args.node != ".": + args.shellcmd[0:0] = [args.node] + args.node = None + + if args.node: + name = args.node + nodedir = rundir.joinpath(name) + if not nodedir.exists(): + print('"{name}" node doesn\'t exist in "{rundir}"') + return 1 + rundir = nodedir + else: + name = "munet" + pidpath = rundir.joinpath("nspid") + pid = open(pidpath, encoding="ascii").read().strip() + + env = {**os.environ} + env["MUNET_NODENAME"] = name + env["MUNET_RUNDIR"] = str(rundir) + + for k in envcfg: + envcfg[k] = envcfg[k].replace("%NAME%", str(name)) + envcfg[k] = envcfg[k].replace("%RUNDIR%", str(rundir)) + + # Can't use -F if it's a new pid namespace + ecmd = "/usr/bin/nsenter" + eargs = [ecmd] + + output = subprocess.check_output(["/usr/bin/nsenter", "--help"], encoding="utf-8") + if " -a," in output: + eargs.append("-a") + else: + # -U doesn't work + for flag in ["-u", "-i", "-m", "-n", "-C", "-T"]: + if f" {flag}," in output: + eargs.append(flag) + eargs.append(f"--pid=/proc/{pid}/ns/pid_for_children") + eargs.append(f"--wd={rundir}") + eargs.extend(["-t", pid]) + eargs += args.shellcmd + # print("Using ", eargs) + return os.execvpe(ecmd, eargs, {**env, **envcfg}) + + +if __name__ == "__main__": + exit_status = main() + sys.exit(exit_status) diff --git a/tests/topotests/munet/mulog.py b/tests/topotests/munet/mulog.py new file mode 100644 index 0000000..f840eae --- /dev/null +++ b/tests/topotests/munet/mulog.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# December 4 2022, Christian Hopps <chopps@labn.net> +# +# Copyright (c) 2022, LabN Consulting, L.L.C. +# +"""Utilities for logging in munet.""" + +import logging + +from pathlib import Path + + +class MultiFileHandler(logging.FileHandler): + """A logging handler that logs to new files based on the logger name. + + The MultiFileHandler operates as a FileHandler with additional functionality. In + addition to logging to the specified logging file MultiFileHandler also creates new + FileHandlers for child loggers based on a root logging name path. + + The ``root_path`` determines when to create a new FileHandler. For each received log + record, ``root_path`` is removed from the logger name of the record if present, and + the resulting channel path (if any) determines the directory for a new log file to + also emit the record to. The new file path is constructed by starting with the + directory ``filename`` resides in, then joining the path determined above after + converting "." to "/" and finally by adding back the basename of ``filename``. + + record logger path => mutest.output.testingfoo + root_path => mutest.output + base filename => /tmp/mutest/mutest-exec.log + new logfile => /tmp/mutest/testingfoo/mutest-exec.log + + All messages are also emitted to the common FileLogger for ``filename``. + + If a log record is from a logger that does not start with ``root_path`` no file is + created and the normal emit occurs. + + Args: + root_path: the logging path of the root level for this handler. + new_handler_level: logging level for newly created handlers + log_dir: the log directory to put log files in. + filename: the base log file. + """ + + def __init__(self, root_path, filename=None, **kwargs): + self.__root_path = root_path + self.__basename = Path(filename).name + if root_path[-1] != ".": + self.__root_path += "." + self.__root_pathlen = len(self.__root_path) + self.__kwargs = kwargs + self.__log_dir = Path(filename).absolute().parent + self.__log_dir.mkdir(parents=True, exist_ok=True) + self.__filenames = {} + self.__added = set() + + if "new_handler_level" not in kwargs: + self.__new_handler_level = logging.NOTSET + else: + new_handler_level = kwargs["new_handler_level"] + del kwargs["new_handler_level"] + self.__new_handler_level = new_handler_level + + super().__init__(filename=filename, **kwargs) + + if self.__new_handler_level is None: + self.__new_handler_level = self.level + + def __log_filename(self, name): + if name in self.__filenames: + return self.__filenames[name] + + if not name.startswith(self.__root_path): + newname = None + else: + newname = name[self.__root_pathlen :] + newname = Path(newname.replace(".", "/")) + newname = self.__log_dir.joinpath(newname) + newname = newname.joinpath(self.__basename) + self.__filenames[name] = newname + + self.__filenames[name] = newname + return newname + + def emit(self, record): + newname = self.__log_filename(record.name) + if newname: + if newname not in self.__added: + self.__added.add(newname) + h = logging.FileHandler(filename=newname, **self.__kwargs) + h.setLevel(self.__new_handler_level) + h.setFormatter(self.formatter) + logging.getLogger(record.name).addHandler(h) + h.emit(record) + super().emit(record) + + +class ColorFormatter(logging.Formatter): + """A formatter that adds color sequences based on level.""" + + def __init__(self, fmt=None, datefmt=None, style="%", **kwargs): + grey = "\x1b[90m" + yellow = "\x1b[33m" + red = "\x1b[31m" + bold_red = "\x1b[31;1m" + reset = "\x1b[0m" + # basefmt = " ------| %(message)s " + + self.formatters = { + logging.DEBUG: logging.Formatter(grey + fmt + reset), + logging.INFO: logging.Formatter(grey + fmt + reset), + logging.WARNING: logging.Formatter(yellow + fmt + reset), + logging.ERROR: logging.Formatter(red + fmt + reset), + logging.CRITICAL: logging.Formatter(bold_red + fmt + reset), + } + # Why are we even bothering? + super().__init__(fmt, datefmt, style, **kwargs) + + def format(self, record): + formatter = self.formatters.get(record.levelno) + return formatter.format(record) diff --git a/tests/topotests/munet/munet-schema.json b/tests/topotests/munet/munet-schema.json new file mode 100644 index 0000000..a1dcd87 --- /dev/null +++ b/tests/topotests/munet/munet-schema.json @@ -0,0 +1,654 @@ +{ + "title": "labn-munet-config", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "description": "Generated by pyang from module labn-munet-config", + "type": "object", + "properties": { + "cli": { + "type": "object", + "properties": { + "commands": { + "type": "array", + "items": { + "type": "object", + "properties": { + "exec": { + "type": "string" + }, + "exec-kind": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "exec": { + "type": "string" + } + } + } + }, + "format": { + "type": "string" + }, + "help": { + "type": "string" + }, + "interactive": { + "type": "boolean" + }, + "kinds": { + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "type": "string" + }, + "new-window": { + "type": "boolean" + }, + "top-level": { + "type": "boolean" + } + } + } + } + } + }, + "kinds": { + "type": "array", + "items": { + "type": "object", + "properties": { + "merge": { + "type": "array", + "items": { + "type": "string" + } + }, + "cap-add": { + "type": "array", + "items": { + "type": "string" + } + }, + "cap-remove": { + "type": "array", + "items": { + "type": "string" + } + }, + "cmd": { + "type": "string" + }, + "cleanup-cmd": { + "type": "string" + }, + "ready-cmd": { + "type": "string" + }, + "image": { + "type": "string" + }, + "server": { + "type": "string" + }, + "server-port": { + "type": "number" + }, + "qemu": { + "type": "object", + "properties": { + "bios": { + "type": "string" + }, + "disk": { + "type": "string" + }, + "kerenel": { + "type": "string" + }, + "initrd": { + "type": "string" + }, + "kvm": { + "type": "boolean" + }, + "ncpu": { + "type": "integer" + }, + "memory": { + "type": "string" + }, + "root": { + "type": "string" + }, + "cmdline-extra": { + "type": "string" + }, + "extra-args": { + "type": "string" + }, + "console": { + "type": "object", + "properties": { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "expects": { + "type": "array", + "items": { + "type": "string" + } + }, + "sends": { + "type": "array", + "items": { + "type": "string" + } + }, + "timeout": { + "type": "integer" + } + } + } + } + }, + "connections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "to": { + "type": "string" + }, + "ip": { + "type": "string" + }, + "ipv6": { + "type": "string" + }, + "name": { + "type": "string" + }, + "hostintf": { + "type": "string" + }, + "physical": { + "type": "string" + }, + "remote-name": { + "type": "string" + }, + "driver": { + "type": "string" + }, + "delay": { + "type": "integer" + }, + "jitter": { + "type": "integer" + }, + "jitter-correlation": { + "type": "string" + }, + "loss": { + "type": "integer" + }, + "loss-correlation": { + "type": "string" + }, + "rate": { + "type": "object", + "properties": { + "rate": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "limit": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "burst": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + } + } + } + } + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + }, + "gdb-cmd": { + "type": "string" + }, + "gdb-target-cmds": { + "type": "array", + "items": { + "type": "string" + } + }, + "gdb-run-cmds": { + "type": "array", + "items": { + "type": "string" + } + }, + "init": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ] + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "destination": { + "type": "string" + }, + "source": { + "type": "string" + }, + "tmpfs-size": { + "type": "string" + }, + "type": { + "type": "string" + } + } + } + }, + "name": { + "type": "string" + }, + "podman": { + "type": "object", + "properties": { + "extra-args": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "privileged": { + "type": "boolean" + }, + "shell": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ] + }, + "volumes": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "topology": { + "type": "object", + "properties": { + "dns-network": { + "type": "string" + }, + "ipv6-enable": { + "type": "boolean" + }, + "networks-autonumber": { + "type": "boolean" + }, + "networks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "ip": { + "type": "string" + }, + "ipv6": { + "type": "string" + } + } + } + }, + "nodes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "kind": { + "type": "string" + }, + "cap-add": { + "type": "array", + "items": { + "type": "string" + } + }, + "cap-remove": { + "type": "array", + "items": { + "type": "string" + } + }, + "cmd": { + "type": "string" + }, + "cleanup-cmd": { + "type": "string" + }, + "ready-cmd": { + "type": "string" + }, + "image": { + "type": "string" + }, + "server": { + "type": "string" + }, + "server-port": { + "type": "number" + }, + "qemu": { + "type": "object", + "properties": { + "bios": { + "type": "string" + }, + "disk": { + "type": "string" + }, + "kerenel": { + "type": "string" + }, + "initrd": { + "type": "string" + }, + "kvm": { + "type": "boolean" + }, + "ncpu": { + "type": "integer" + }, + "memory": { + "type": "string" + }, + "root": { + "type": "string" + }, + "cmdline-extra": { + "type": "string" + }, + "extra-args": { + "type": "string" + }, + "console": { + "type": "object", + "properties": { + "user": { + "type": "string" + }, + "password": { + "type": "string" + }, + "expects": { + "type": "array", + "items": { + "type": "string" + } + }, + "sends": { + "type": "array", + "items": { + "type": "string" + } + }, + "timeout": { + "type": "integer" + } + } + } + } + }, + "connections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "to": { + "type": "string" + }, + "ip": { + "type": "string" + }, + "ipv6": { + "type": "string" + }, + "name": { + "type": "string" + }, + "hostintf": { + "type": "string" + }, + "physical": { + "type": "string" + }, + "remote-name": { + "type": "string" + }, + "driver": { + "type": "string" + }, + "delay": { + "type": "integer" + }, + "jitter": { + "type": "integer" + }, + "jitter-correlation": { + "type": "string" + }, + "loss": { + "type": "integer" + }, + "loss-correlation": { + "type": "string" + }, + "rate": { + "type": "object", + "properties": { + "rate": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "limit": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + }, + "burst": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ] + } + } + } + } + } + }, + "env": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + }, + "gdb-cmd": { + "type": "string" + }, + "gdb-target-cmds": { + "type": "array", + "items": { + "type": "string" + } + }, + "gdb-run-cmds": { + "type": "array", + "items": { + "type": "string" + } + }, + "init": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ] + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "destination": { + "type": "string" + }, + "source": { + "type": "string" + }, + "tmpfs-size": { + "type": "string" + }, + "type": { + "type": "string" + } + } + } + }, + "name": { + "type": "string" + }, + "podman": { + "type": "object", + "properties": { + "extra-args": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "privileged": { + "type": "boolean" + }, + "shell": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ] + }, + "volumes": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + }, + "version": { + "type": "integer" + } + } +}
\ No newline at end of file diff --git a/tests/topotests/munet/mutest/__main__.py b/tests/topotests/munet/mutest/__main__.py new file mode 100644 index 0000000..c870311 --- /dev/null +++ b/tests/topotests/munet/mutest/__main__.py @@ -0,0 +1,445 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# December 2 2022, Christian Hopps <chopps@labn.net> +# +# Copyright (c) 2022, LabN Consulting, L.L.C. +# +"""Command to execute mutests.""" + +import asyncio +import logging +import os +import subprocess +import sys +import time + +from argparse import ArgumentParser +from argparse import Namespace +from copy import deepcopy +from pathlib import Path +from typing import Union + +from munet import parser +from munet.base import Bridge +from munet.base import get_event_loop +from munet.mutest import userapi as uapi +from munet.native import L3NodeMixin +from munet.native import Munet +from munet.parser import async_build_topology +from munet.parser import get_config + + +# We want all but critical to fit in 5 characters for alignment +logging.addLevelName(logging.WARNING, "WARN") +root_logger = logging.getLogger("") +exec_formatter = logging.Formatter("%(asctime)s %(levelname)5s: %(name)s: %(message)s") + + +async def get_unet(config: dict, croot: Path, rundir: Path, unshare: bool = False): + """Create and run a new Munet topology. + + The topology is built from the given ``config`` to run inside the path indicated + by ``rundir``. If ``unshare`` is True then the process will unshare into it's + own private namespace. + + Args: + config: a config dictionary obtained from ``munet.parser.get_config``. This + value will be modified and stored in the built ``Munet`` object. + croot: common root of all tests, used to search for ``kinds.yaml`` files. + rundir: the path to the run directory for this topology. + unshare: True to unshare the process into it's own private namespace. + + Yields: + Munet: The constructed and running topology. + """ + tasks = [] + unet = None + try: + try: + unet = await async_build_topology( + config, rundir=str(rundir), unshare_inline=unshare + ) + except Exception as error: + logging.debug("unet build failed: %s", error, exc_info=True) + raise + try: + tasks = await unet.run() + except Exception as error: + logging.debug("unet run failed: %s", error, exc_info=True) + raise + logging.debug("unet topology running") + try: + yield unet + except Exception as error: + logging.error("unet fixture: yield unet unexpected exception: %s", error) + raise + except KeyboardInterrupt: + logging.info("Received keyboard while building topology") + raise + finally: + if unet: + await unet.async_delete() + + # No one ever awaits these so cancel them + logging.debug("unet fixture: cleanup") + for task in tasks: + task.cancel() + + # Reset the class variables so auto number is predictable + logging.debug("unet fixture: resetting ords to 1") + L3NodeMixin.next_ord = 1 + Bridge.next_ord = 1 + + +def common_root(path1: Union[str, Path], path2: Union[str, Path]) -> Path: + """Find the common root between 2 paths. + + Args: + path1: Path + path2: Path + Returns: + Path: the shared root components between ``path1`` and ``path2``. + + Examples: + >>> common_root("/foo/bar/baz", "/foo/bar/zip/zap") + PosixPath('/foo/bar') + >>> common_root("/foo/bar/baz", "/fod/bar/zip/zap") + PosixPath('/') + """ + apath1 = Path(path1).absolute().parts + apath2 = Path(path2).absolute().parts + alen = min(len(apath1), len(apath2)) + common = None + for a, b in zip(apath1[:alen], apath2[:alen]): + if a != b: + break + common = common.joinpath(a) if common else Path(a) + return common + + +async def collect(args: Namespace): + """Collect test files. + + Files must match the pattern ``mutest_*.py``, and their containing + directory must have a munet config file present. This function also changes + the current directory to the common parent of all the tests, and paths are + returned relative to the common directory. + + Args: + args: argparse results + + Returns: + (commondir, tests, configs): where ``commondir`` is the path representing + the common parent directory of all the testsd, ``tests`` is a + dictionary of lists of test files, keyed on their containing directory + path, and ``configs`` is a dictionary of config dictionaries also keyed + on its containing directory path. The directory paths are relative to a + common ancestor. + """ + file_select = args.file_select + upaths = args.paths if args.paths else ["."] + globpaths = set() + for upath in (Path(x) for x in upaths): + if upath.is_file(): + paths = {upath.absolute()} + else: + paths = {x.absolute() for x in Path(upath).rglob(file_select)} + globpaths |= paths + tests = {} + configs = {} + + # Find the common root + # We don't actually need this anymore, the idea was prefix test names + # with uncommon paths elements to automatically differentiate them. + common = None + sortedpaths = [] + for path in sorted(globpaths): + sortedpaths.append(path) + dirpath = path.parent + common = common_root(common, dirpath) if common else dirpath + + ocwd = Path().absolute() + try: + os.chdir(common) + # Work with relative paths to the common directory + for path in (x.relative_to(common) for x in sortedpaths): + dirpath = path.parent + if dirpath not in configs: + try: + configs[dirpath] = get_config(search=[dirpath]) + except FileNotFoundError: + logging.warning( + "Skipping '%s' as munet.{yaml,toml,json} not found in '%s'", + path, + dirpath, + ) + continue + if dirpath not in tests: + tests[dirpath] = [] + tests[dirpath].append(path.absolute()) + finally: + os.chdir(ocwd) + return common, tests, configs + + +async def execute_test( + unet: Munet, + test: Path, + args: Namespace, + test_num: int, + exec_handler: logging.Handler, +) -> (int, int, int, Exception): + """Execute a test case script. + + Using the built and running topology in ``unet`` for targets + execute the test case script file ``test``. + + Args: + unet: a running topology. + test: path to the test case script file. + args: argparse results. + test_num: the number of this test case in the run. + exec_handler: exec file handler to add to test loggers which do not propagate. + """ + test_name = testname_from_path(test) + + # Get test case loggers + logger = logging.getLogger(f"mutest.output.{test_name}") + reslog = logging.getLogger(f"mutest.results.{test_name}") + logger.addHandler(exec_handler) + reslog.addHandler(exec_handler) + + # We need to send an info level log to cause the speciifc handler to be + # created, otherwise all these debug ones don't get through + reslog.info("") + + # reslog.debug("START: %s:%s from %s", test_num, test_name, test.stem) + # reslog.debug("-" * 70) + + targets = dict(unet.hosts.items()) + targets["."] = unet + + tc = uapi.TestCase( + str(test_num), test_name, test, targets, logger, reslog, args.full_summary + ) + passed, failed, e = tc.execute() + + run_time = time.time() - tc.info.start_time + + status = "PASS" if not (failed or e) else "FAIL" + + # Turn off for now + reslog.debug("-" * 70) + reslog.debug( + "stats: %d steps, %d pass, %d fail, %s abort, %4.2fs elapsed", + passed + failed, + passed, + failed, + 1 if e else 0, + run_time, + ) + reslog.debug("-" * 70) + reslog.debug("END: %s %s:%s\n", status, test_num, test_name) + + return passed, failed, e + + +def testname_from_path(path: Path) -> str: + """Return test name based on the path to the test file. + + Args: + path: path to the test file. + + Returns: + str: the name of the test. + """ + return str(Path(path).stem).replace("/", ".") + + +def print_header(reslog, unet): + targets = dict(unet.hosts.items()) + nmax = max(len(x) for x in targets) + nmax = max(nmax, len("TARGET")) + sum_fmt = uapi.TestCase.sum_fmt.format(nmax) + reslog.info(sum_fmt, "NUMBER", "STAT", "TARGET", "TIME", "DESCRIPTION") + reslog.info("-" * 70) + + +async def run_tests(args): + reslog = logging.getLogger("mutest.results") + + common, tests, configs = await collect(args) + results = [] + errlog = logging.getLogger("mutest.error") + reslog = logging.getLogger("mutest.results") + printed_header = False + tnum = 0 + start_time = time.time() + try: + for dirpath in tests: + test_files = tests[dirpath] + for test in test_files: + tnum += 1 + config = deepcopy(configs[dirpath]) + test_name = testname_from_path(test) + rundir = args.rundir.joinpath(test_name) + + # Add an test case exec file handler to the root logger and result + # logger + exec_path = rundir.joinpath("mutest-exec.log") + exec_path.parent.mkdir(parents=True, exist_ok=True) + exec_handler = logging.FileHandler(exec_path, "w") + exec_handler.setFormatter(exec_formatter) + root_logger.addHandler(exec_handler) + + try: + async for unet in get_unet(config, common, rundir): + if not printed_header: + print_header(reslog, unet) + printed_header = True + passed, failed, e = await execute_test( + unet, test, args, tnum, exec_handler + ) + except KeyboardInterrupt as error: + errlog.warning("KeyboardInterrupt while running test %s", test_name) + passed, failed, e = 0, 0, error + raise + except Exception as error: + logging.error( + "Error executing test %s: %s", test, error, exc_info=True + ) + errlog.error( + "Error executing test %s: %s", test, error, exc_info=True + ) + passed, failed, e = 0, 0, error + finally: + # Remove the test case exec file handler form the root logger. + root_logger.removeHandler(exec_handler) + results.append((test_name, passed, failed, e)) + + except KeyboardInterrupt: + pass + + run_time = time.time() - start_time + tnum = 0 + tpassed = 0 + tfailed = 0 + texc = 0 + + spassed = 0 + sfailed = 0 + + for result in results: + _, passed, failed, e = result + tnum += 1 + spassed += passed + sfailed += failed + if e: + texc += 1 + if failed or e: + tfailed += 1 + else: + tpassed += 1 + + reslog.info("") + reslog.info( + "run stats: %s steps, %s pass, %s fail, %s abort, %4.2fs elapsed", + spassed + sfailed, + spassed, + sfailed, + texc, + run_time, + ) + reslog.info("-" * 70) + + tnum = 0 + for result in results: + test_name, passed, failed, e = result + tnum += 1 + s = "FAIL" if failed or e else "PASS" + reslog.info(" %s %s:%s", s, tnum, test_name) + + reslog.info("-" * 70) + reslog.info( + "END RUN: %s test scripts, %s passed, %s failed", tnum, tpassed, tfailed + ) + + return 1 if tfailed else 0 + + +async def async_main(args): + status = 3 + try: + # For some reson we are not catching exceptions raised inside + status = await run_tests(args) + except KeyboardInterrupt: + logging.info("Exiting (async_main), received KeyboardInterrupt in main") + except Exception as error: + logging.info( + "Exiting (async_main), unexpected exception %s", error, exc_info=True + ) + logging.debug("async_main returns %s", status) + return status + + +def main(): + ap = ArgumentParser() + ap.add_argument( + "--dist", + type=int, + nargs="?", + const=-1, + default=0, + action="store", + metavar="NUM-THREADS", + help="Run in parallel, value is num. of threads or no value for auto", + ) + ap.add_argument("-d", "--rundir", help="runtime directory for tempfiles, logs, etc") + ap.add_argument( + "--file-select", default="mutest_*.py", help="shell glob for finding tests" + ) + ap.add_argument("--log-config", help="logging config file (yaml, toml, json, ...)") + ap.add_argument( + "-V", + "--full-summary", + action="store_true", + help="print full summary headers from docstrings", + ) + ap.add_argument( + "-v", dest="verbose", action="count", default=0, help="More -v's, more verbose" + ) + ap.add_argument("paths", nargs="*", help="Paths to collect tests from") + args = ap.parse_args() + + rundir = args.rundir if args.rundir else "/tmp/mutest" + args.rundir = Path(rundir) + os.environ["MUNET_RUNDIR"] = rundir + subprocess.run(f"mkdir -p {rundir} && chmod 755 {rundir}", check=True, shell=True) + + config = parser.setup_logging(args, config_base="logconf-mutest") + # Grab the exec formatter from the logging config + if fconfig := config.get("formatters", {}).get("exec"): + global exec_formatter # pylint: disable=W291,W0603 + exec_formatter = logging.Formatter( + fconfig.get("format"), fconfig.get("datefmt") + ) + + loop = None + status = 4 + try: + loop = get_event_loop() + status = loop.run_until_complete(async_main(args)) + except KeyboardInterrupt: + logging.info("Exiting (main), received KeyboardInterrupt in main") + except Exception as error: + logging.info("Exiting (main), unexpected exception %s", error, exc_info=True) + finally: + if loop: + loop.close() + + sys.exit(status) + + +if __name__ == "__main__": + main() diff --git a/tests/topotests/munet/mutest/userapi.py b/tests/topotests/munet/mutest/userapi.py new file mode 100644 index 0000000..1df8c0d --- /dev/null +++ b/tests/topotests/munet/mutest/userapi.py @@ -0,0 +1,1111 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright 2017, 2022, LabN Consulting, L.L.C. +"""Mutest is a simple send/expect based testing framework. + +This module implements the basic send/expect functionality for mutest. The test +developer first creates a munet topology (:ref:`munet-config`) and then writes test +scripts ("test cases") which are composed of calls to the functions defined below +("steps"). In short these are: + +Send/Expect functions: + + - :py:func:`step` + + - :py:func:`step_json` + + - :py:func:`match_step` + + - :py:func:`match_step_json` + + - :py:func:`wait_step` + + - :py:func:`wait_step_json` + +Control/Utility functions: + + - :py:func:`script_dir` + + - :py:func:`include` + + - :py:func:`log` + + - :py:func:`test` + +Test scripts are located by the :command:`mutest` command by their name. The name of a +test script should take the form ``mutest_TESTNAME.py`` where ``TESTNAME`` is replaced +with a user chosen name for the test case. + +Here's a simple example test script which first checks that a specific forwarding entry +is in the FIB for the IP destination ``10.0.1.1``. Then it checks repeatedly for up to +10 seconds for a second forwarding entry in the FIB for the IP destination ``10.0.2.1``. + +.. code-block:: python + + match_step("r1", 'vtysh -c "show ip fib 10.0.1.1"', "Routing entry for 10.0.1.0/24", + "Check for FIB entry for 10.0.1.1") + + wait_step("r1", + 'vtysh -c "show ip fib 10.0.2.1"', + "Routing entry for 10.0.2.0/24", + desc="Check for FIB entry for 10.0.2.1", + timeout=10) + +Notice that the call arguments can be specified by their correct position in the list or +using keyword names, and they can also be specified over multiple lines if preferred. + +All of the functions are documented and defined below. +""" + +# pylint: disable=global-statement + +import functools +import json +import logging +import pprint +import re +import time + +from pathlib import Path +from typing import Any +from typing import Union + +from deepdiff import DeepDiff as json_cmp + +from munet.base import Commander + + +class TestCaseInfo: + """Object to hold nestable TestCase Results.""" + + def __init__(self, tag: str, name: str, path: Path): + self.path = path.absolute() + self.tag = tag + self.name = name + self.steps = 0 + self.passed = 0 + self.failed = 0 + self.start_time = time.time() + self.step_start_time = self.start_time + self.run_time = None + + def __repr__(self): + return ( + f"TestCaseInfo({self.tag} {self.name} steps {self.steps} " + f"p {self.passed} f {self.failed} path {self.path})" + ) + + +class TestCase: + """A mutest testcase. + + This is normally meant to be used internally by the mutest command to + implement the user API. See README-mutest.org for usage details on the + user API. + + Args: + tag: identity of the test in a run. (x.x...) + name: the name of the test case + path: the test file that is being executed. + targets: a dictionary of objects which implement ``cmd_nostatus(str)`` + output_logger: a logger for output and other messages from the test. + result_logger: a logger to output the results of test steps to. + full_summary: if True then print entire doctstring instead of + only the first line in the results report + + Attributes: + tag: identity of the test in a run + name: the name of the test + targets: dictionary of targets. + + steps: total steps executed so far. + passed: number of passing steps. + failed: number of failing steps. + + last: the last command output. + last_m: the last result of re.search during a matching step on the output with + newlines converted to spaces. + + :meta private: + """ + + # sum_hfmt = "{:5.5s} {:4.4s} {:>6.6s} {}" + # sum_dfmt = "{:5s} {:4.4s} {:^6.6s} {}" + sum_fmt = "%-8.8s %4.4s %{}s %6s %s" + + def __init__( + self, + tag: int, + name: str, + path: Path, + targets: dict, + output_logger: logging.Logger = None, + result_logger: logging.Logger = None, + full_summary: bool = False, + ): + + self.info = TestCaseInfo(tag, name, path) + self.__saved_info = [] + self.__short_doc_header = not full_summary + + self.__space_before_result = False + + # we are only ever in a section once, an include ends a section + # so are never in section+include, and another section ends a + # section, so we don't need __in_section to be save in the + # TestCaseInfo struct. + self.__in_section = False + + self.targets = targets + + self.last = "" + self.last_m = None + + self.rlog = result_logger + self.olog = output_logger + self.logf = functools.partial(self.olog.log, logging.INFO) + + oplog = logging.getLogger("mutest.oper") + self.oplogf = oplog.debug + self.oplogf("new TestCase: tag: %s name: %s path: %s", tag, name, path) + + # find the longerst target name and make target field that wide + nmax = max(len(x) for x in targets) + nmax = max(nmax, len("TARGET")) + self.sum_fmt = TestCase.sum_fmt.format(nmax) + + # Let's keep this out of summary for now + self.rlog.debug(self.sum_fmt, "NUMBER", "STAT", "TARGET", "TIME", "DESCRIPTION") + self.rlog.debug("-" * 70) + + @property + def tag(self): + return self.info.tag + + @property + def name(self): + return self.info.name + + @property + def steps(self): + return self.info.steps + + @property + def passed(self): + return self.info.passed + + @property + def failed(self): + return self.info.failed + + def execute(self): + """Execute the test case. + + :meta private: + """ + assert TestCase.g_tc is None + self.oplogf("execute") + try: + TestCase.g_tc = self + e = self.__exec_script(self.info.path, True, False) + except BaseException: + self.__end_test() + raise + return *self.__end_test(), e + + def __del__(self): + if TestCase.g_tc is self: + logging.error("Internal error, TestCase.__end_test() was not called!") + TestCase.g_tc = None + + def __push_execinfo(self, path: Path): + self.oplogf( + "__push_execinfo: path: %s current top is %s", + path, + pprint.pformat(self.info), + ) + newname = self.name + path.stem + self.info.steps += 1 + self.__saved_info.append(self.info) + tag = f"{self.info.tag}.{self.info.steps}" + self.info = TestCaseInfo(tag, newname, path) + self.oplogf("__push_execinfo: now on top: %s", pprint.pformat(self.info)) + + def __pop_execinfo(self): + # do something with tag? + finished_info = self.info + self.info = self.__saved_info.pop() + self.oplogf(" __pop_execinfo: poppped: %s", pprint.pformat(finished_info)) + self.oplogf(" __pop_execinfo: now on top: %s", pprint.pformat(self.info)) + return finished_info + + def __print_header(self, tag, header, add_newline=False): + # self.olog.info(self.sum_fmt, tag, "", "", "", header) + self.olog.info("== %s ==", f"TEST: {tag}. {header}") + if add_newline: + self.rlog.info("") + self.rlog.info("%s. %s", tag, header) + + def __exec_script(self, path, print_header, add_newline): + + # Below was the original method to avoid the global TestCase + # variable; however, we need global functions so we can import them + # into test scripts. Without imports pylint will complain about undefined + # functions and the resulting christmas tree of warnings is annoying. + # + # pylint: disable=possibly-unused-variable,exec-used,redefined-outer-name + # include = self.include + # log = self.logf + # match_step = self.match_step + # match_step_json = self.match_step_json + # step = self.step + # step_json = self.step_json + # test = self.test + # wait_step = self.wait_step + # wait_step_json = self.wait_step_json + + name = f"{path.stem}{self.tag}" + name = re.sub(r"\W|^(?=\d)", "_", name) + + _ok_result = "marker" + try: + self.oplogf("__exec_script: path %s", path) + script = open(path, "r", encoding="utf-8").read() + + # Load the script into a function. + script = script.strip() + s2 = ( + # f"async def _{name}(ok_result):\n" + f"def _{name}(ok_result):\n" + + " " + + script.replace("\n", "\n ") + + "\n return ok_result\n" + + "\n" + ) + exec(s2) + + # Extract any docstring as a title. + if print_header: + title = locals()[f"_{name}"].__doc__.lstrip() + if self.__short_doc_header and (title := title.lstrip()): + if (idx := title.find("\n")) != -1: + title = title[:idx].strip() + if not title: + title = f"Test from file: {self.info.path.name}" + self.__print_header(self.info.tag, title, add_newline) + self.__space_before_result = False + + # Execute the function. + result = locals()[f"_{name}"](_ok_result) + + # Here's where we can do async in the future if we want. + # result = await locals()[f"_{name}"](_ok_result) + except Exception as error: + logging.error( + "Unexpected exception executing %s: %s", name, error, exc_info=True + ) + return error + else: + if result is not _ok_result: + logging.info("%s returned early, result: %s", name, result) + else: + self.oplogf("__exec_script: name %s completed normally", name) + return None + + def __post_result(self, target, success, rstr, logstr=None): + self.oplogf( + "__post_result: target: %s success %s rstr %s", target, success, rstr + ) + if success: + self.info.passed += 1 + status = "PASS" + outlf = self.logf + reslf = self.rlog.info + else: + self.info.failed += 1 + status = "FAIL" + outlf = self.olog.warning + reslf = self.rlog.warning + + self.info.steps += 1 + if logstr is not None: + outlf("R:%d %s: %s" % (self.steps, status, logstr)) + + run_time = time.time() - self.info.step_start_time + + stepstr = f"{self.tag}.{self.steps}" + rtimes = _delta_time_str(run_time) + + if self.__space_before_result: + self.rlog.info("") + self.__space_before_result = False + + reslf(self.sum_fmt, stepstr, status, target, rtimes, rstr) + + # start counting for next step now + self.info.step_start_time = time.time() + + def __end_test(self) -> (int, int): + """End the test log final results. + + Returns: + number of steps, number passed, number failed, run time. + """ + self.oplogf("__end_test: __in_section: %s", self.__in_section) + if self.__in_section: + self.__end_section() + + passed, failed = self.info.passed, self.info.failed + + # No close for loggers + # self.olog.close() + # self.rlog.close() + self.olog = None + self.rlog = None + + assert ( + TestCase.g_tc == self + ), "TestCase global unexpectedly someon else in __end_test" + TestCase.g_tc = None + + self.info.run_time = time.time() - self.info.start_time + return passed, failed + + def _command( + self, + target: str, + cmd: str, + ) -> str: + """Execute a ``cmd`` and return result. + + Args: + target: the target to execute the command on. + cmd: string to execut on the target. + """ + out = self.targets[target].cmd_nostatus(cmd, warn=False) + self.last = out = out.rstrip() + report = out if out else "<no output>" + self.logf("COMMAND OUTPUT:\n%s", report) + return out + + def _command_json( + self, + target: str, + cmd: str, + ) -> dict: + """Execute a json ``cmd`` and return json result. + + Args: + target: the target to execute the command on. + cmd: string to execut on the target. + """ + out = self.targets[target].cmd_nostatus(cmd, warn=False) + self.last = out = out.rstrip() + try: + js = json.loads(out) + except Exception as error: + js = {} + self.olog.warning( + "JSON load failed. Check command output is in JSON format: %s", + error, + ) + self.logf("COMMAND OUTPUT:\n%s", out) + return js + + def _match_command( + self, + target: str, + cmd: str, + match: str, + expect_fail: bool, + flags: int, + ) -> (bool, Union[str, list]): + """Execute a ``cmd`` and check result. + + Args: + target: the target to execute the command on. + cmd: string to execute on the target. + match: regex to ``re.search()`` for in output. + expect_fail: if True then succeed when the regexp doesn't match. + flags: python regex flags to modify matching behavior + + Returns: + (success, matches): if the match fails then "matches" will be None, + otherwise if there were matching groups then groups() will be returned in + ``matches`` otherwise group(0) (i.e., the matching text). + """ + out = self._command(target, cmd) + search = re.search(match, out, flags) + self.last_m = search + if search is None: + success = expect_fail + ret = None + else: + success = not expect_fail + ret = search.groups() + if not ret: + ret = search.group(0) + + level = logging.DEBUG if success else logging.WARNING + self.olog.log(level, "matched:%s:", ret) + return success, ret + + def _match_command_json( + self, + target: str, + cmd: str, + match: Union[str, dict], + expect_fail: bool, + ) -> Union[str, dict]: + """Execute a json ``cmd`` and check result. + + Args: + target: the target to execute the command on. + cmd: string to execut on the target. + match: A json ``str`` or object (``dict``) to compare against the json + output from ``cmd``. + expect_fail: if True then succeed when the json doesn't match. + """ + js = self._command_json(target, cmd) + try: + expect = json.loads(match) + except Exception as error: + expect = {} + self.olog.warning( + "JSON load failed. Check match value is in JSON format: %s", error + ) + + if json_diff := json_cmp(expect, js): + success = expect_fail + if not success: + self.logf("JSON DIFF:%s:" % json_diff) + return success, json_diff + + success = not expect_fail + return success, js + + def _wait( + self, + target: str, + cmd: str, + match: Union[str, dict], + is_json: bool, + timeout: float, + interval: float, + expect_fail: bool, + flags: int, + ) -> Union[str, dict]: + """Execute a command repeatedly waiting for result until timeout.""" + startt = time.time() + endt = startt + timeout + + success = False + ret = None + while not success and time.time() < endt: + if is_json: + success, ret = self._match_command_json(target, cmd, match, expect_fail) + else: + success, ret = self._match_command( + target, cmd, match, expect_fail, flags + ) + if not success: + time.sleep(interval) + return success, ret + + # --------------------- + # Public APIs for User + # --------------------- + + def include(self, pathname: str, new_section: bool = False): + """See :py:func:`~munet.mutest.userapi.include`. + + :meta private: + """ + path = Path(pathname) + path = self.info.path.parent.joinpath(path) + + self.oplogf( + "include: new path: %s create section: %s currently __in_section: %s", + path, + new_section, + self.__in_section, + ) + + if new_section: + self.oplogf("include: starting new exec section") + self.__start_exec_section(path) + our_info = self.info + # Note we do *not* mark __in_section True + else: + # swap the current path inside the top info + old_path = self.info.path + self.info.path = path + self.oplogf("include: swapped info path: new %s old %s", path, old_path) + + self.__exec_script(path, print_header=new_section, add_newline=new_section) + + if new_section: + # Something within the section creating include has also created a section + # end it, sections do not cross section creating file boundaries + if self.__in_section: + self.oplogf( + "include done: path: %s __in_section calling __end_section", path + ) + self.__end_section() + + # We should now be back to the info we started with, b/c we don't actually + # start a new section (__in_section) that then could have been ended inside + # the included file. + assert our_info == self.info + + self.oplogf( + "include done: path: %s new_section calling __end_section", path + ) + self.__end_section() + else: + # The current top path could be anything due to multiple inline includes as + # well as section swap in and out. Forcibly return the top path to the file + # we are returning to + self.info.path = old_path + self.oplogf("include: restored info path: %s", old_path) + + def __end_section(self): + self.oplogf("__end_section: __in_section: %s", self.__in_section) + info = self.__pop_execinfo() + passed, failed = info.passed, info.failed + self.info.passed += passed + self.info.failed += failed + self.__space_before_result = True + self.oplogf("__end_section setting __in_section to False") + self.__in_section = False + + def __start_exec_section(self, path): + self.oplogf("__start_exec_section: __in_section: %s", self.__in_section) + if self.__in_section: + self.__end_section() + + self.__push_execinfo(path) + self.__space_before_result = False + self.oplogf("NOT setting __in_section to True") + assert not self.__in_section + + def section(self, desc: str): + """See :py:func:`~munet.mutest.userapi.section`. + + :meta private: + """ + self.oplogf("section: __in_section: %s", self.__in_section) + # Grab path before we pop the current info off the top + path = self.info.path + old_steps = self.info.steps + + if self.__in_section: + self.__end_section() + + self.__push_execinfo(path) + add_nl = self.info.steps <= old_steps + + self.__space_before_result = False + self.__in_section = True + self.oplogf(" section setting __in_section to True") + self.__print_header(self.info.tag, desc, add_nl) + + def step(self, target: str, cmd: str) -> str: + """See :py:func:`~munet.mutest.userapi.step`. + + :meta private: + """ + self.logf( + "#%s.%s:%s:STEP:%s:%s", + self.tag, + self.steps + 1, + self.info.path, + target, + cmd, + ) + return self._command(target, cmd) + + def step_json(self, target: str, cmd: str) -> dict: + """See :py:func:`~munet.mutest.userapi.step_json`. + + :meta private: + """ + self.logf( + "#%s.%s:%s:STEP_JSON:%s:%s", + self.tag, + self.steps + 1, + self.info.path, + target, + cmd, + ) + return self._command_json(target, cmd) + + def match_step( + self, + target: str, + cmd: str, + match: str, + desc: str = "", + expect_fail: bool = False, + flags: int = re.DOTALL, + ) -> (bool, Union[str, list]): + """See :py:func:`~munet.mutest.userapi.match_step`. + + :meta private: + """ + self.logf( + "#%s.%s:%s:MATCH_STEP:%s:%s:%s:%s:%s:%s", + self.tag, + self.steps + 1, + self.info.path, + target, + cmd, + match, + desc, + expect_fail, + flags, + ) + success, ret = self._match_command(target, cmd, match, expect_fail, flags) + if desc: + self.__post_result(target, success, desc) + return success, ret + + def test_step(self, expr_or_value: Any, desc: str, target: str = "") -> bool: + """See :py:func:`~munet.mutest.userapi.test`. + + :meta private: + """ + success = bool(expr_or_value) + self.__post_result(target, success, desc) + return success + + def match_step_json( + self, + target: str, + cmd: str, + match: Union[str, dict], + desc: str = "", + expect_fail: bool = False, + ) -> (bool, Union[str, dict]): + """See :py:func:`~munet.mutest.userapi.match_step_json`. + + :meta private: + """ + self.logf( + "#%s.%s:%s:MATCH_STEP_JSON:%s:%s:%s:%s:%s", + self.tag, + self.steps + 1, + self.info.path, + target, + cmd, + match, + desc, + expect_fail, + ) + success, ret = self._match_command_json(target, cmd, match, expect_fail) + if desc: + self.__post_result(target, success, desc) + return success, ret + + def wait_step( + self, + target: str, + cmd: str, + match: Union[str, dict], + desc: str = "", + timeout=10, + interval=0.5, + expect_fail: bool = False, + flags: int = re.DOTALL, + ) -> (bool, Union[str, list]): + """See :py:func:`~munet.mutest.userapi.wait_step`. + + :meta private: + """ + if interval is None: + interval = min(timeout / 20, 0.25) + self.logf( + "#%s.%s:%s:WAIT_STEP:%s:%s:%s:%s:%s:%s:%s:%s", + self.tag, + self.steps + 1, + self.info.path, + target, + cmd, + match, + timeout, + interval, + desc, + expect_fail, + flags, + ) + success, ret = self._wait( + target, cmd, match, False, timeout, interval, expect_fail, flags + ) + if desc: + self.__post_result(target, success, desc) + return success, ret + + def wait_step_json( + self, + target: str, + cmd: str, + match: Union[str, dict], + desc: str = "", + timeout=10, + interval=None, + expect_fail: bool = False, + ) -> (bool, Union[str, dict]): + """See :py:func:`~munet.mutest.userapi.wait_step_json`. + + :meta private: + """ + if interval is None: + interval = min(timeout / 20, 0.25) + self.logf( + "#%s.%s:%s:WAIT_STEP:%s:%s:%s:%s:%s:%s:%s", + self.tag, + self.steps + 1, + self.info.path, + target, + cmd, + match, + timeout, + interval, + desc, + expect_fail, + ) + success, ret = self._wait( + target, cmd, match, True, timeout, interval, expect_fail, 0 + ) + if desc: + self.__post_result(target, success, desc) + return success, ret + + +# A non-rentrant global to allow for simplified operations +TestCase.g_tc = None + +# pylint: disable=protected-access + + +def _delta_time_str(run_time: float) -> str: + if run_time < 0.0001: + return "0.0" + if run_time < 0.001: + return f"{run_time:1.4f}" + if run_time < 0.01: + return f"{run_time:2.3f}" + if run_time < 0.1: + return f"{run_time:3.2f}" + if run_time < 100: + return f"{run_time:4.1f}" + return f"{run_time:5f}s" + + +def section(desc: str): + """Start a new section for steps, with a description. + + This starts a new section of tests. The result is basically + the same as doing a non-inline include. The current test number + is used to form a new sub-set of test steps. So if the current + test number is 2.3, a section will now number subsequent steps + 2.3.1, 2.3.2, ... + + A subsequent :py:func:`section` or non-inline :py:func:`include` + call ends the current section and advances the base test number. + + Args: + desc: the description for the new section. + """ + TestCase.g_tc.section(desc) + + +def log(fmt, *args, **kwargs): + """Log a message in the testcase output log.""" + return TestCase.g_tc.logf(fmt, *args, **kwargs) + + +def include(pathname: str, new_section=False): + """Include a file as part of testcase. + + Args: + pathname: the file to include. + new_section: if a new section should be created, otherwise + commands are executed inline. + """ + return TestCase.g_tc.include(pathname, new_section) + + +def script_dir() -> Path: + """The pathname to the directory containing the current script file. + + When an include() is called the script_dir is updated to be current with the + includeded file, and is reverted to the previous value when the include completes. + """ + return TestCase.g_tc.info.path.parent + + +def get_target(name: str) -> Commander: + """Get the target object with the given ``name``.""" + return TestCase.g_tc.targets[name] + + +def step(target: str, cmd: str) -> str: + """Execute a ``cmd`` on a ``target`` and return the output. + + Args: + target: the target to execute the ``cmd`` on. + cmd: string to execute on the target. + + Returns: + Returns the ``str`` output of the ``cmd``. + """ + return TestCase.g_tc.step(target, cmd) + + +def step_json(target: str, cmd: str) -> dict: + """Execute a json ``cmd`` on a ``target`` and return the json object. + + Args: + target: the target to execute the ``cmd`` on. + cmd: string to execute on the target. + + Returns: + Returns the json object after parsing the ``cmd`` output. + + If json parse fails, a warning is logged and an empty ``dict`` is used. + """ + return TestCase.g_tc.step_json(target, cmd) + + +def test_step(expr_or_value: Any, desc: str, target: str = "") -> bool: + """Evaluates ``expr_or_value`` and posts a result base on it bool(expr). + + If ``expr_or_value`` evaluates to a positive result (i.e., True, non-zero, non-None, + non-empty string, non-empty list, etc..) then a PASS result is recorded, otherwise + record a FAIL is recorded. + + Args: + expr: an expression or value to evaluate + desc: description of this test step. + target: optional target to associate with this test in the result string. + + Returns: + A bool indicating the test PASS or FAIL result. + """ + return TestCase.g_tc.test_step(expr_or_value, desc, target) + + +def match_step( + target: str, + cmd: str, + match: str, + desc: str = "", + expect_fail: bool = False, + flags: int = re.DOTALL, +) -> (bool, Union[str, list]): + """Execute a ``cmd`` on a ``target`` check result. + + Execute ``cmd`` on ``target`` and check if the regexp in ``match`` + matches or doesn't match (according to the ``expect_fail`` value) the + ``cmd`` output. + + If the ``match`` regexp includes groups and if the match succeeds + the group values will be returned in a list, otherwise the command + output is returned. + + Args: + target: the target to execute the ``cmd`` on. + cmd: string to execut on the ``target``. + match: regex to match against output. + desc: description of test, if no description then no result is logged. + expect_fail: if True then succeed when the regexp doesn't match. + flags: python regex flags to modify matching behavior + + Returns: + Returns a 2-tuple. The first value is a bool indicating ``success``. + The second value will be a list from ``re.Match.groups()`` if non-empty, + otherwise ``re.Match.group(0)`` if there was a match otherwise None. + """ + return TestCase.g_tc.match_step(target, cmd, match, desc, expect_fail, flags) + + +def match_step_json( + target: str, + cmd: str, + match: Union[str, dict], + desc: str = "", + expect_fail: bool = False, +) -> (bool, Union[str, dict]): + """Execute a ``cmd`` on a ``target`` check result. + + Execute ``cmd`` on ``target`` and check if the json object in ``match`` + matches or doesn't match (according to the ``expect_fail`` value) the + json output from ``cmd``. + + Args: + target: the target to execute the ``cmd`` on. + cmd: string to execut on the ``target``. + match: A json ``str`` or object (``dict``) to compare against the json + output from ``cmd``. + desc: description of test, if no description then no result is logged. + expect_fail: if True then succeed if the a json doesn't match. + + Returns: + Returns a 2-tuple. The first value is a bool indicating ``success``. The + second value is a ``str`` diff if there is a difference found in the json + compare, otherwise the value is the json object (``dict``) from the ``cmd``. + + If json parse fails, a warning is logged and an empty ``dict`` is used. + """ + return TestCase.g_tc.match_step_json(target, cmd, match, desc, expect_fail) + + +def wait_step( + target: str, + cmd: str, + match: Union[str, dict], + desc: str = "", + timeout: float = 10.0, + interval: float = 0.5, + expect_fail: bool = False, + flags: int = re.DOTALL, +) -> (bool, Union[str, list]): + """Execute a ``cmd`` on a ``target`` repeatedly, looking for a result. + + Execute ``cmd`` on ``target``, every ``interval`` seconds for up to ``timeout`` + seconds until the output of ``cmd`` does or doesn't match (according to the + ``expect_fail`` value) the ``match`` value. + + Args: + target: the target to execute the ``cmd`` on. + cmd: string to execut on the ``target``. + match: regexp to match against output. + timeout: The number of seconds to repeat the ``cmd`` looking for a match + (or non-match if ``expect_fail`` is True). + interval: The number of seconds between running the ``cmd``. If not + specified the value is calculated from the timeout value so that on + average the cmd will execute 10 times. The minimum calculated interval + is .25s, shorter values can be passed explicitly. + desc: description of test, if no description then no result is logged. + expect_fail: if True then succeed when the regexp *doesn't* match. + flags: python regex flags to modify matching behavior + + Returns: + Returns a 2-tuple. The first value is a bool indicating ``success``. + The second value will be a list from ``re.Match.groups()`` if non-empty, + otherwise ``re.Match.group(0)`` if there was a match otherwise None. + """ + return TestCase.g_tc.wait_step( + target, cmd, match, desc, timeout, interval, expect_fail, flags + ) + + +def wait_step_json( + target: str, + cmd: str, + match: Union[str, dict], + desc: str = "", + timeout=10, + interval=None, + expect_fail: bool = False, +) -> (bool, Union[str, dict]): + """Execute a cmd repeatedly and wait for matching result. + + Execute ``cmd`` on ``target``, every ``interval`` seconds until + the output of ``cmd`` matches or doesn't match (according to the + ``expect_fail`` value) ``match``, for up to ``timeout`` seconds. + + ``match`` is a regular expression to search for in the output of ``cmd`` when + ``is_json`` is False. + + When ``is_json`` is True ``match`` must be a json object or a ``str`` which + parses into a json object. Likewise, the ``cmd`` output is parsed into a json + object and then a comparison is done between the two json objects. + + Args: + target: the target to execute the ``cmd`` on. + cmd: string to execut on the ``target``. + match: A json object or str representation of one to compare against json + output from ``cmd``. + desc: description of test, if no description then no result is logged. + timeout: The number of seconds to repeat the ``cmd`` looking for a match + (or non-match if ``expect_fail`` is True). + interval: The number of seconds between running the ``cmd``. If not + specified the value is calculated from the timeout value so that on + average the cmd will execute 10 times. The minimum calculated interval + is .25s, shorter values can be passed explicitly. + expect_fail: if True then succeed if the a json doesn't match. + + Returns: + Returns a 2-tuple. The first value is a bool indicating ``success``. + The second value is a ``str`` diff if there is a difference found in the + json compare, otherwise the value is a json object (dict) from the ``cmd`` + output. + + If json parse fails, a warning is logged and an empty ``dict`` is used. + """ + return TestCase.g_tc.wait_step_json( + target, cmd, match, desc, timeout, interval, expect_fail + ) + + +def luInclude(filename, CallOnFail=None): + """Backward compatible API, do not use in new tests.""" + return include(filename) + + +def luLast(usenl=False): + """Backward compatible API, do not use in new tests.""" + del usenl + return TestCase.g_tc.last_m + + +def luCommand( + target, + cmd, + regexp=".", + op="none", + result="", + ltime=10, + returnJson=False, + wait_time=0.5, +): + """Backward compatible API, do not use in new tests. + + Only non-json is verified to any degree of confidence by code inspection. + + For non-json should return match.group() if match else return bool(op == "fail"). + + For json if no diff return the json else diff return bool(op == "jsoncmp_fail") + bug if no json from output (fail parse) could maybe generate diff, which could + then return + """ + if op == "wait": + if returnJson: + return wait_step_json(target, cmd, regexp, result, ltime, wait_time) + + success, _ = wait_step(target, cmd, regexp, result, ltime, wait_time) + match = luLast() + if success and match is not None: + return match.group() + return success + + if op == "none": + if returnJson: + return step_json(target, cmd) + return step(target, cmd) + + if returnJson and op in ("jsoncmp_fail", "jsoncmp_pass"): + expect_fail = op == "jsoncmp_fail" + return match_step_json(target, cmd, regexp, result, expect_fail) + + assert not returnJson + assert op in ("fail", "pass") + expect_fail = op == "fail" + success, _ = match_step(target, cmd, regexp, result, expect_fail) + match = luLast() + if success and match is not None: + return match.group() + return success diff --git a/tests/topotests/munet/mutestshare.py b/tests/topotests/munet/mutestshare.py new file mode 100644 index 0000000..95ffa74 --- /dev/null +++ b/tests/topotests/munet/mutestshare.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# January 28 2023, Christian Hopps <chopps@labn.net> +# +# Copyright (c) 2023, LabN Consulting, L.L.C. +# +"""A tiny init for namespaces in python inspired by the C program tini.""" +import argparse +import errno +import logging +import os +import shlex +import signal +import subprocess +import sys +import threading +import time + +from signal import Signals as S + +from . import linux +from .base import commander + + +child_pid = -1 +very_verbose = False +restore_signals = set() + + +def vdebug(*args, **kwargs): + if very_verbose: + logging.debug(*args, **kwargs) + + +def exit_with_status(pid, status): + try: + ec = status >> 8 if bool(status & 0xFF00) else status | 0x80 + logging.debug("reaped our child, exiting %s", ec) + sys.exit(ec) + except ValueError: + vdebug("pid %s didn't actually exit", pid) + + +def waitpid(tag): + logging.debug("%s: waitid for exiting processes", tag) + idobj = os.waitid(os.P_ALL, 0, os.WEXITED) + pid = idobj.si_pid + status = idobj.si_status + if pid == child_pid: + exit_with_status(pid, status) + else: + logging.debug("%s: reaped zombie pid %s with status %s", tag, pid, status) + + +def new_process_group(): + pid = os.getpid() + try: + pgid = os.getpgrp() + if pgid == pid: + logging.debug("already process group leader %s", pgid) + else: + logging.debug("creating new process group %s", pid) + os.setpgid(pid, 0) + except Exception as error: + logging.warning("unable to get new process group: %s", error) + return + + # Block these in order to allow foregrounding, otherwise we'd get SIGTTOU blocked + signal.signal(S.SIGTTIN, signal.SIG_IGN) + signal.signal(S.SIGTTOU, signal.SIG_IGN) + fd = sys.stdin.fileno() + if not os.isatty(fd): + logging.debug("stdin not a tty no foregrounding required") + else: + try: + # This will error if our session no longer associated with controlling tty. + pgid = os.tcgetpgrp(fd) + if pgid == pid: + logging.debug("process group already in foreground %s", pgid) + else: + logging.debug("making us the foreground pgid backgrounding %s", pgid) + os.tcsetpgrp(fd, pid) + except OSError as error: + if error.errno == errno.ENOTTY: + logging.debug("session is no longer associated with controlling tty") + else: + logging.warning("unable to foreground pgid %s: %s", pid, error) + signal.signal(S.SIGTTIN, signal.SIG_DFL) + signal.signal(S.SIGTTOU, signal.SIG_DFL) + + +def exec_child(exec_args): + # Restore signals to default handling: + for snum in restore_signals: + signal.signal(snum, signal.SIG_DFL) + + # Create new process group. + new_process_group() + + estring = shlex.join(exec_args) + try: + # and exec the process + logging.debug("child: executing '%s'", estring) + os.execvp(exec_args[0], exec_args) + # NOTREACHED + except Exception as error: + logging.warning("child: unable to execute '%s': %s", estring, error) + raise + + +def is_creating_pid_namespace(): + p1name = subprocess.check_output( + "readlink /proc/self/pid", stderr=subprocess.STDOUT, shell=True + ) + p2name = subprocess.check_output( + "readlink /proc/self/pid_for_children", stderr=subprocess.STDOUT, shell=True + ) + return p1name != p2name + + +def restore_namespace(ppid_fd, uflags): + fd = ppid_fd + retry = 3 + for i in range(0, retry): + try: + linux.setns(fd, uflags) + except OSError as error: + logging.warning("could not reset to old namespace fd %s: %s", fd, error) + if i == retry - 1: + raise + time.sleep(1) + os.close(fd) + + +def create_thread_test(): + def runthread(name): + logging.info("In thread: %s", name) + + logging.info("Create thread") + thread = threading.Thread(target=runthread, args=(1,)) + logging.info("Run thread") + thread.start() + logging.info("Join thread") + thread.join() + + +def run(args): + del args + # We look for this b/c the unshare pid will share with /sibn/init + # nselm = "pid_for_children" + # nsflags.append(f"--pid={pp / nselm}") + # mutini now forks when created this way + # cmd.append("--pid") + # cmd.append("--fork") + # cmd.append("--kill-child") + # cmd.append("--mount-proc") + + uflags = linux.CLONE_NEWPID + nslist = ["pid_for_children"] + uflags |= linux.CLONE_NEWNS + nslist.append("mnt") + uflags |= linux.CLONE_NEWNET + nslist.append("net") + + # Before values + pid = os.getpid() + nsdict = {x: os.readlink(f"/tmp/mu-global-proc/{pid}/ns/{x}") for x in nslist} + + # + # UNSHARE + # + create_thread_test() + + ppid = os.getppid() + ppid_fd = linux.pidfd_open(ppid) + linux.unshare(uflags) + + # random syscall's fail until we fork a child to establish the new pid namespace. + global child_pid # pylint: disable=global-statement + child_pid = os.fork() + if not child_pid: + logging.info("In child sleeping") + time.sleep(1200) + sys.exit(1) + + # verify after values differ + nnsdict = {x: os.readlink(f"/tmp/mu-global-proc/{pid}/ns/{x}") for x in nslist} + assert not {k for k in nsdict if nsdict[k] == nnsdict[k]} + + # Remount / and any future mounts below it as private + commander.cmd_raises("mount --make-rprivate /") + # Mount a new /proc in our new namespace + commander.cmd_raises("mount -t proc proc /proc") + + # + # In NEW NS + # + + cid = os.fork() + if not cid: + logging.info("In second child sleeping") + time.sleep(4) + sys.exit(1) + logging.info("Waiting for second child") + os.waitpid(cid, 0) + + try: + create_thread_test() + except Exception as error: + print(error) + + # + # RESTORE + # + + logging.info("In new namespace, restoring old") + # Make sure we can go back, not sure since this is PID namespace, but maybe + restore_namespace(ppid_fd, uflags) + + # verify after values the same + nnsdict = {x: os.readlink(f"/proc/self/ns/{x}") for x in nslist} + assert nsdict == nnsdict + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "-v", dest="verbose", action="count", default=0, help="More -v's, more verbose" + ) + ap.add_argument("rest", nargs=argparse.REMAINDER) + args = ap.parse_args() + + level = logging.DEBUG if args.verbose else logging.INFO + if args.verbose > 1: + global very_verbose # pylint: disable=global-statement + very_verbose = True + logging.basicConfig( + level=level, format="%(asctime)s mutini: %(levelname)s: %(message)s" + ) + + status = 4 + try: + run(args) + except KeyboardInterrupt: + logging.info("exiting (main), received KeyboardInterrupt in main") + except Exception as error: + logging.info("exiting (main), unexpected exception %s", error, exc_info=True) + + sys.exit(status) + + +if __name__ == "__main__": + main() diff --git a/tests/topotests/munet/mutini.py b/tests/topotests/munet/mutini.py new file mode 100755 index 0000000..e5f9931 --- /dev/null +++ b/tests/topotests/munet/mutini.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# January 28 2023, Christian Hopps <chopps@labn.net> +# +# Copyright (c) 2023, LabN Consulting, L.L.C. +# +"""A tiny init for namespaces in python inspired by the C program tini.""" + + +# pylint: disable=global-statement +import argparse +import errno +import logging +import os +import re +import shlex +import signal +import subprocess +import sys + +from signal import Signals as S + + +try: + from munet import linux +except ModuleNotFoundError: + # We cannot use relative imports and still run this module directly as a script, and + # there are some use cases where we want to run this file as a script. + sys.path.append(os.path.dirname(os.path.realpath(__file__))) + import linux + + +class g: + """Global variables for our program.""" + + child_pid = -1 + orig_pid = os.getpid() + exit_signal = False + pid_status_cache = {} + restore_signals = set() + very_verbose = False + + +unshare_flags = { + "C": linux.CLONE_NEWCGROUP, + "i": linux.CLONE_NEWIPC, + "m": linux.CLONE_NEWNS, + "n": linux.CLONE_NEWNET, + "p": linux.CLONE_NEWPID, + "u": linux.CLONE_NEWUTS, + "T": linux.CLONE_NEWTIME, +} + + +ignored_signals = { + S.SIGTTIN, + S.SIGTTOU, +} +abort_signals = { + S.SIGABRT, + S.SIGBUS, + S.SIGFPE, + S.SIGILL, + S.SIGKILL, + S.SIGSEGV, + S.SIGSTOP, + S.SIGSYS, + S.SIGTRAP, +} +no_prop_signals = abort_signals | ignored_signals | {S.SIGCHLD} + + +def vdebug(*args, **kwargs): + if g.very_verbose: + logging.debug(*args, **kwargs) + + +def get_pid_status_item(status, stat): + m = re.search(rf"(?:^|\n){stat}:\t(.*)(?:\n|$)", status) + return m.group(1).strip() if m else None + + +def pget_pid_status_item(pid, stat): + if pid not in g.pid_status_cache: + with open(f"/proc/{pid}/status", "r", encoding="utf-8") as f: + g.pid_status_cache[pid] = f.read().strip() + return get_pid_status_item(g.pid_status_cache[pid], stat).strip() + + +def get_pid_name(pid): + try: + return get_pid_status_item(g.pid_status_cache[pid], "Name") + except Exception: + return str(pid) + + +# def init_get_child_pids(): +# """Return list of "children" pids. +# We consider any process with a 0 parent pid to also be our child as it +# nsentered our pid namespace from an external parent. +# """ +# g.pid_status_cache.clear() +# pids = (int(x) for x in os.listdir("/proc") if x.isdigit() and x != "1") +# return ( +# x for x in pids if x == g.child_pid or pget_pid_status_item(x, "PPid") == "0" +# ) + + +def exit_with_status(status): + if os.WIFEXITED(status): + ec = os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + ec = 0x80 | os.WTERMSIG(status) + else: + ec = 255 + logging.debug("exiting with code %s", ec) + sys.exit(ec) + + +def waitpid(tag): + logging.debug("%s: waitid for exiting process", tag) + idobj = os.waitid(os.P_ALL, 0, os.WEXITED) + pid = idobj.si_pid + status = idobj.si_status + + if pid != g.child_pid: + pidname = get_pid_name(pid) + logging.debug( + "%s: reaped zombie %s (%s) w/ status %s", tag, pid, pidname, status + ) + return + + logging.debug("reaped child with status %s", status) + exit_with_status(status) + # NOTREACHED + + +def sig_trasmit(signum, _): + signame = signal.Signals(signum).name + if g.child_pid == -1: + # We've received a signal after setting up to be init proc + # but prior to fork or fork returning with child pid + logging.debug("received %s prior to child exec, exiting", signame) + sys.exit(0x80 | signum) + + try: + os.kill(g.child_pid, signum) + except OSError as error: + if error.errno != errno.ESRCH: + logging.error( + "error forwarding signal %s to child, exiting: %s", signum, error + ) + sys.exit(0x80 | signum) + logging.debug("child pid %s exited prior to signaling", g.child_pid) + + +def sig_sigchld(signum, _): + assert signum == S.SIGCHLD + try: + waitpid("SIGCHLD") + except ChildProcessError as error: + logging.warning("got SIGCHLD but no pid to wait on: %s", error) + + +def setup_init_signals(): + valid = set(signal.valid_signals()) + named = set(x.value for x in signal.Signals) + for snum in sorted(named): + if snum not in valid: + continue + if S.SIGRTMIN <= snum <= S.SIGRTMAX: + continue + + sname = signal.Signals(snum).name + if snum == S.SIGCHLD: + vdebug("installing local handler for %s", sname) + signal.signal(snum, sig_sigchld) + g.restore_signals.add(snum) + elif snum in ignored_signals: + vdebug("installing ignore handler for %s", sname) + signal.signal(snum, signal.SIG_IGN) + g.restore_signals.add(snum) + elif snum in abort_signals: + vdebug("leaving default handler for %s", sname) + # signal.signal(snum, signal.SIG_DFL) + else: + vdebug("installing trasmit signal handler for %s", sname) + try: + signal.signal(snum, sig_trasmit) + g.restore_signals.add(snum) + except OSError as error: + logging.warning( + "failed installing signal handler for %s: %s", sname, error + ) + + +def new_process_group(): + """Create and lead a new process group. + + This function will create a new process group if we are not yet leading one, and + additionally foreground said process group in our session. This foregrounding + action is copied from tini, and I believe serves a purpose when serving as init + for a container (e.g., podman). + """ + pid = os.getpid() + try: + pgid = os.getpgrp() + if pgid == pid: + logging.debug("already process group leader %s", pgid) + else: + logging.debug("creating new process group %s", pid) + os.setpgid(pid, 0) + except Exception as error: + logging.warning("unable to get new process group: %s", error) + return + + # Block these in order to allow foregrounding, otherwise we'd get SIGTTOU blocked + signal.signal(S.SIGTTIN, signal.SIG_IGN) + signal.signal(S.SIGTTOU, signal.SIG_IGN) + fd = sys.stdin.fileno() + if not os.isatty(fd): + logging.debug("stdin not a tty no foregrounding required") + else: + try: + # This will error if our session no longer associated with controlling tty. + pgid = os.tcgetpgrp(fd) + if pgid == pid: + logging.debug("process group already in foreground %s", pgid) + else: + logging.debug("making us the foreground pgid backgrounding %s", pgid) + os.tcsetpgrp(fd, pid) + except OSError as error: + if error.errno == errno.ENOTTY: + logging.debug("session is no longer associated with controlling tty") + else: + logging.warning("unable to foreground pgid %s: %s", pid, error) + signal.signal(S.SIGTTIN, signal.SIG_DFL) + signal.signal(S.SIGTTOU, signal.SIG_DFL) + + +def is_creating_pid_namespace(): + p1name = subprocess.check_output( + "readlink /proc/self/pid", stderr=subprocess.STDOUT, shell=True + ) + p2name = subprocess.check_output( + "readlink /proc/self/pid_for_children", stderr=subprocess.STDOUT, shell=True + ) + return p1name != p2name + + +def be_init(new_pg, exec_args): + # + # Arrange for us to be killed when our parent dies, this will subsequently also kill + # all procs in any PID namespace we are init for. + # + logging.debug("set us to be SIGKILLed when parent exits") + linux.set_parent_death_signal(signal.SIGKILL) + + # If we are createing a new PID namespace for children... + if g.orig_pid != 1: + logging.debug("started as pid %s", g.orig_pid) + # assert is_creating_pid_namespace() + + # Fork to become pid 1 + logging.debug("forking to become pid 1") + child_pid = os.fork() + if child_pid: + logging.debug("in parent waiting on child pid %s to exit", child_pid) + status = os.wait() + logging.debug("got child exit status %s", status) + exit_with_status(status) + # NOTREACHED + + # We must be pid 1 now. + logging.debug("in child as pid %s", os.getpid()) + assert os.getpid() == 1 + + # We need a new /proc now. + logging.debug("mount new /proc") + linux.mount("proc", "/proc", "proc") + + # If the parent exists kill us using SIGKILL + logging.debug("set us to be SIGKILLed when parent exits") + linux.set_parent_death_signal(signal.SIGKILL) + + if not exec_args: + if not new_pg: + logging.debug("no exec args, no new process group") + # # if 0 == os.getpgid(0): + # status = os.setpgid(0, 1) + # logging.debug("os.setpgid(0, 1) == %s", status) + else: + logging.debug("no exec args, creating new process group") + # No exec so we are the "child". + new_process_group() + + # Reap children as init process + vdebug("installing local handler for SIGCHLD") + signal.signal(signal.SIGCHLD, sig_sigchld) + + while True: + logging.info("init: waiting to reap zombies") + linux.pause() + # NOTREACHED + + # Set (parent) signal handlers before any fork to avoid race + setup_init_signals() + + logging.debug("forking to execute child") + g.child_pid = os.fork() + if g.child_pid == 0: + # In child, restore signals to default handling: + for snum in g.restore_signals: + signal.signal(snum, signal.SIG_DFL) + + # XXX is a new pg right? + new_process_group() + logging.debug("child: executing '%s'", shlex.join(exec_args)) + os.execvp(exec_args[0], exec_args) + # NOTREACHED + + while True: + logging.info("parent: waiting for child pid %s to exit", g.child_pid) + waitpid("parent") + + +def unshare(flags): + """Unshare into new namespaces.""" + uflags = 0 + for flag in flags: + if flag not in unshare_flags: + raise ValueError(f"unknown unshare flag '{flag}'") + uflags |= unshare_flags[flag] + new_pid = bool(uflags & linux.CLONE_NEWPID) + new_mnt = bool(uflags & linux.CLONE_NEWNS) + + logging.debug("unshareing with flags: %s", linux.clone_flag_string(uflags)) + linux.unshare(uflags) + + if new_pid and not new_mnt: + try: + # If we are not creating new mount namspace, remount /proc private + # so that our mount of a new /proc doesn't affect parent namespace + logging.debug("remount /proc recursive private") + linux.mount("none", "/proc", None, linux.MS_REC | linux.MS_PRIVATE) + except OSError as error: + # EINVAL is OK b/c /proc not mounted may cause an error + if error.errno != errno.EINVAL: + raise + if new_mnt: + # Remount root as recursive private. + logging.debug("remount / recursive private") + linux.mount("none", "/", None, linux.MS_REC | linux.MS_PRIVATE) + + # if new_pid: + # logging.debug("mount new /proc") + # linux.mount("proc", "/proc", "proc") + + return new_pid + + +def main(): + # + # Parse CLI args. + # + + ap = argparse.ArgumentParser() + ap.add_argument( + "-P", + "--no-proc-group", + action="store_true", + help="set to inherit the process group", + ) + valid_flags = "".join(unshare_flags) + ap.add_argument( + "--unshare-flags", + help=( + f"string of unshare(1) flags. Supported values from '{valid_flags}'." + " 'm' will remount `/` recursive private. 'p' will remount /proc" + " and fork, and the child will be signaled to exit on exit of parent.." + ), + ) + ap.add_argument( + "-v", dest="verbose", action="count", default=0, help="more -v's, more verbose" + ) + ap.add_argument("rest", nargs=argparse.REMAINDER) + args = ap.parse_args() + + # + # Setup logging. + # + + level = logging.DEBUG if args.verbose else logging.INFO + if args.verbose > 1: + g.very_verbose = True + logging.basicConfig( + level=level, format="%(asctime)s mutini: %(levelname)s: %(message)s" + ) + + # + # Run program + # + + status = 5 + try: + new_pid = False + if args.unshare_flags: + new_pid = unshare(args.unshare_flags) + + if g.orig_pid != 1 and not new_pid: + # Simply hold the namespaces + while True: + logging.info("holding namespace waiting to be signaled to exit") + linux.pause() + # NOTREACHED + + be_init(not args.no_proc_group, args.rest) + # NOTREACHED + logging.critical("Exited from be_init!") + except KeyboardInterrupt: + logging.info("exiting (main), received KeyboardInterrupt in main") + status = 0x80 | signal.SIGINT + except Exception as error: + logging.info("exiting (main), do to exception %s", error, exc_info=True) + + sys.exit(status) + + +if __name__ == "__main__": + main() diff --git a/tests/topotests/munet/native.py b/tests/topotests/munet/native.py new file mode 100644 index 0000000..fecf709 --- /dev/null +++ b/tests/topotests/munet/native.py @@ -0,0 +1,2941 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# October 1 2021, Christian Hopps <chopps@labn.net> +# +# Copyright (c) 2021-2022, LabN Consulting, L.L.C. +# +# pylint: disable=protected-access +"""A module that defines objects for standalone use.""" +import asyncio +import errno +import getpass +import ipaddress +import logging +import os +import random +import re +import shlex +import socket +import subprocess +import time + +from . import cli +from .base import BaseMunet +from .base import Bridge +from .base import Commander +from .base import LinuxNamespace +from .base import MunetError +from .base import Timeout +from .base import _async_get_exec_path +from .base import _get_exec_path +from .base import cmd_error +from .base import commander +from .base import fsafe_name +from .base import get_exec_path_host +from .config import config_subst +from .config import config_to_dict_with_key +from .config import find_matching_net_config +from .config import find_with_kv +from .config import merge_kind_config + + +class L3ContainerNotRunningError(MunetError): + """Exception if no running container exists.""" + + +def get_loopback_ips(c, nid): + if ip := c.get("ip"): + if ip == "auto": + return [ipaddress.ip_interface("10.255.0.0/32") + nid] + if isinstance(ip, str): + return [ipaddress.ip_interface(ip)] + return [ipaddress.ip_interface(x) for x in ip] + return [] + + +def make_ip_network(net, inc): + n = ipaddress.ip_network(net) + return ipaddress.ip_network( + (n.network_address + inc * n.num_addresses, n.prefixlen) + ) + + +def make_ip_interface(ia, inc): + ia = ipaddress.ip_interface(ia) + # this turns into a /32 fix this + ia = ia + ia.network.num_addresses * inc + # IPv6 + ia = ipaddress.ip_interface(str(ia).replace("/32", "/24").replace("/128", "/64")) + return ia + + +def get_ip_network(c, brid, ipv6=False): + ip = c.get("ipv6" if ipv6 else "ip") + if ip and str(ip) != "auto": + try: + ifip = ipaddress.ip_interface(ip) + if ifip.ip == ifip.network.network_address: + return ifip.network + return ifip + except ValueError: + return ipaddress.ip_network(ip) + if ipv6: + return make_ip_interface("fc00::fe/64", brid) + return make_ip_interface("10.0.0.254/24", brid) + + +def parse_pciaddr(devaddr): + comp = re.match( + "(?:([0-9A-Fa-f]{4}):)?([0-9A-Fa-f]{2}):([0-9A-Fa-f]{2}).([0-7])", devaddr + ).groups() + if comp[0] is None: + comp[0] = "0000" + return [int(x, 16) for x in comp] + + +def read_int_value(path): + return int(open(path, encoding="ascii").read()) + + +def read_str_value(path): + return open(path, encoding="ascii").read().strip() + + +def read_sym_basename(path): + return os.path.basename(os.readlink(path)) + + +async def to_thread(func): + """to_thread for python < 3.9.""" + try: + return await asyncio.to_thread(func) + except AttributeError: + logging.warning("Using backport to_thread") + return await asyncio.get_running_loop().run_in_executor(None, func) + + +def convert_ranges_to_bitmask(ranges): + bitmask = 0 + for r in ranges.split(","): + if "-" not in r: + bitmask |= 1 << int(r) + else: + x, y = (int(x) for x in r.split("-")) + for b in range(x, y + 1): + bitmask |= 1 << b + return bitmask + + +class L2Bridge(Bridge): + """A linux bridge with no IP network address.""" + + def __init__(self, name=None, unet=None, logger=None, mtu=None, config=None): + """Create a linux Bridge.""" + super().__init__(name=name, unet=unet, logger=logger, mtu=mtu) + + self.config = config if config else {} + + async def _async_delete(self): + self.logger.debug("%s: deleting", self) + await super()._async_delete() + + +class L3Bridge(Bridge): + """A linux bridge with associated IP network address.""" + + def __init__(self, name=None, unet=None, logger=None, mtu=None, config=None): + """Create a linux Bridge.""" + super().__init__(name=name, unet=unet, logger=logger, mtu=mtu) + + self.config = config if config else {} + + self.ip_interface = get_ip_network(self.config, self.id) + if hasattr(self.ip_interface, "network"): + self.ip_address = self.ip_interface.ip + self.ip_network = self.ip_interface.network + self.cmd_raises(f"ip addr add {self.ip_interface} dev {name}") + else: + self.ip_address = None + self.ip_network = self.ip_interface + + self.logger.debug("%s: set IPv4 network address to %s", self, self.ip_interface) + self.cmd_raises("sysctl -w net.ipv4.ip_forward=1") + + self.ip6_interface = None + if self.unet.ipv6_enable: + self.ip6_interface = get_ip_network(self.config, self.id, ipv6=True) + if hasattr(self.ip6_interface, "network"): + self.ip6_address = self.ip6_interface.ip + self.ip6_network = self.ip6_interface.network + self.cmd_raises(f"ip addr add {self.ip6_interface} dev {name}") + else: + self.ip6_address = None + self.ip6_network = self.ip6_interface + + self.logger.debug( + "%s: set IPv6 network address to %s", self, self.ip_interface + ) + self.cmd_raises("sysctl -w net.ipv6.conf.all.forwarding=1") + + self.is_nat = self.config.get("nat", False) + if self.is_nat: + self.cmd_raises( + "iptables -t nat -A POSTROUTING " + f"-s {self.ip_network} ! -d {self.ip_network} " + f"! -o {self.name} -j MASQUERADE" + ) + + def get_intf_addr(self, ifname, ipv6=False): + # None is a valid interface, we have the same address for all interfaces + # just make sure they aren't asking for something we don't have. + if ifname is not None and ifname not in self.intfs: + return None + return self.ip6_interface if ipv6 else self.ip_interface + + async def _async_delete(self): + self.logger.debug("%s: deleting", self) + + if self.config.get("nat", False): + self.cmd_status( + "iptables -t nat -D POSTROUTING " + f"-s {self.ip_network} ! -d {self.ip_network} " + f"! -o {self.name} -j MASQUERADE" + ) + await super()._async_delete() + + +class NodeMixin: + """Node attributes and functionality.""" + + next_ord = 1 + + @classmethod + def _get_next_ord(cls): + # Do not use `cls` here b/c that makes the variable class specific + n = L3NodeMixin.next_ord + L3NodeMixin.next_ord = n + 1 + return n + + def __init__(self, *args, config=None, **kwargs): + """Create a Node.""" + super().__init__(*args, **kwargs) + + self.config = config if config else {} + config = self.config + + self.id = int(config["id"]) if "id" in config else self._get_next_ord() + + self.cmd_p = None + self.container_id = None + self.cleanup_called = False + + # Clear and create rundir early + assert self.unet is not None + self.rundir = self.unet.rundir.joinpath(self.name) + commander.cmd_raises(f"rm -rf {self.rundir}") + commander.cmd_raises(f"mkdir -p {self.rundir}") + + def _shebang_prep(self, config_key): + cmd = self.config.get(config_key, "").strip() + if not cmd: + return [] + + script_name = fsafe_name(config_key) + + # shell_cmd is a union and can be boolean or string + shell_cmd = self.config.get("shell", "/bin/bash") + if not isinstance(shell_cmd, str): + if shell_cmd: + # i.e., "shell: true" + shell_cmd = "/bin/bash" + else: + # i.e., "shell: false" + shell_cmd = "" + + # If we have a shell_cmd then we create a cleanup_cmds file in run_cmd + # and volume mounted it + if shell_cmd: + # Create cleanup cmd file + cmd = cmd.replace("%CONFIGDIR%", str(self.unet.config_dirname)) + cmd = cmd.replace("%RUNDIR%", str(self.rundir)) + cmd = cmd.replace("%NAME%", str(self.name)) + cmd += "\n" + + # Write out our cleanup cmd file at this time too. + cmdpath = os.path.join(self.rundir, f"{script_name}.shebang") + with open(cmdpath, mode="w+", encoding="utf-8") as cmdfile: + cmdfile.write(f"#!{shell_cmd}\n") + cmdfile.write(cmd) + cmdfile.flush() + commander.cmd_raises(f"chmod 755 {cmdpath}") + + if self.container_id: + # XXX this counts on it being mounted in container, ugly + cmds = [f"/tmp/{script_name}.shebang"] + else: + cmds = [cmdpath] + else: + cmds = [] + if isinstance(cmd, str): + cmds.extend(shlex.split(cmd)) + else: + cmds.extend(cmd) + cmds = [ + x.replace("%CONFIGDIR%", str(self.unet.config_dirname)) for x in cmds + ] + cmds = [x.replace("%RUNDIR%", str(self.rundir)) for x in cmds] + cmds = [x.replace("%NAME%", str(self.name)) for x in cmds] + + return cmds + + async def _async_shebang_cmd(self, config_key, warn=True): + cmds = self._shebang_prep(config_key) + if not cmds: + return 0 + + rc, o, e = await self.async_cmd_status(cmds, warn=warn) + if not rc and warn and (o or e): + self.logger.info( + f"async_shebang_cmd ({config_key}): %s", cmd_error(rc, o, e) + ) + elif rc and warn: + self.logger.warning( + f"async_shebang_cmd ({config_key}): %s", cmd_error(rc, o, e) + ) + else: + self.logger.debug( + f"async_shebang_cmd ({config_key}): %s", cmd_error(rc, o, e) + ) + + return rc + + def has_run_cmd(self) -> bool: + return bool(self.config.get("cmd", "").strip()) + + async def get_proc_child_pid(self, p): + # commander is right for both unshare inline (our proc pidns) + # and non-inline (root pidns). + + # This doesn't work b/c we can't get back to the root pidns + + rootcmd = self.unet.rootcmd + pgrep = rootcmd.get_exec_path("pgrep") + spid = str(p.pid) + for _ in Timeout(4): + if p.returncode is not None: + self.logger.debug("%s: proc %s exited before getting child", self, p) + return None + + rc, o, e = await rootcmd.async_cmd_status( + [pgrep, "-o", "-P", spid], warn=False + ) + if rc == 0: + return int(o.strip()) + + await asyncio.sleep(0.1) + self.logger.debug( + "%s: no child of proc %s: %s", self, p, cmd_error(rc, o, e) + ) + self.logger.warning("%s: timeout getting child pid of proc %s", self, p) + return None + + async def run_cmd(self): + """Run the configured commands for this node.""" + self.logger.debug( + "[rundir %s exists %s]", self.rundir, os.path.exists(self.rundir) + ) + + cmds = self._shebang_prep("cmd") + if not cmds: + return + + stdout = open(os.path.join(self.rundir, "cmd.out"), "wb") + stderr = open(os.path.join(self.rundir, "cmd.err"), "wb") + self.cmd_pid = None + self.cmd_p = await self.async_popen( + cmds, + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + start_new_session=True, # allows us to signal all children to exit + ) + + # If our process is actually the child of an nsenter fetch its pid. + if self.nsenter_fork: + self.cmd_pid = await self.get_proc_child_pid(self.cmd_p) + + self.logger.debug( + "%s: async_popen %s => %s (cmd_pid %s)", + self, + cmds, + self.cmd_p.pid, + self.cmd_pid, + ) + + self.pytest_hook_run_cmd(stdout, stderr) + + return self.cmd_p + + async def _async_cleanup_cmd(self): + """Run the configured cleanup commands for this node. + + This function is called by subclass' async_cleanup_cmd + """ + self.cleanup_called = True + + return await self._async_shebang_cmd("cleanup-cmd") + + def has_cleanup_cmd(self) -> bool: + return bool(self.config.get("cleanup-cmd", "").strip()) + + async def async_cleanup_cmd(self): + """Run the configured cleanup commands for this node.""" + return await self._async_cleanup_cmd() + + def has_ready_cmd(self) -> bool: + return bool(self.config.get("ready-cmd", "").strip()) + + async def async_ready_cmd(self): + """Run the configured ready commands for this node.""" + return not await self._async_shebang_cmd("ready-cmd", warn=False) + + def cmd_completed(self, future): + self.logger.debug("%s: cmd completed callback", self) + try: + status = future.result() + self.logger.debug( + "%s: node cmd_p completed result: %s cmd: %s", self, status, self.cmd_p + ) + self.cmd_pid = None + self.cmd_p = None + except asyncio.CancelledError: + # Should we stop the container if we have one? + self.logger.debug("%s: node cmd_p.wait() canceled", future) + + def pytest_hook_run_cmd(self, stdout, stderr): + """Handle pytest options related to running the node cmd. + + This function does things such as launch tail'ing windows + on the given files if requested by the user. + + Args: + stdout: file-like object with a ``name`` attribute, or a path to a file. + stderr: file-like object with a ``name`` attribute, or a path to a file. + """ + if not self.unet: + return + + outopt = self.unet.cfgopt.getoption("--stdout") + outopt = outopt if outopt is not None else "" + if outopt == "all" or self.name in outopt.split(","): + outname = stdout.name if hasattr(stdout, "name") else stdout + self.run_in_window(f"tail -F {outname}", title=f"O:{self.name}") + + if stderr: + erropt = self.unet.cfgopt.getoption("--stderr") + erropt = erropt if erropt is not None else "" + if erropt == "all" or self.name in erropt.split(","): + errname = stderr.name if hasattr(stderr, "name") else stderr + self.run_in_window(f"tail -F {errname}", title=f"E:{self.name}") + + def pytest_hook_open_shell(self): + if not self.unet: + return + + gdbcmd = self.config.get("gdb-cmd") + shellopt = self.unet.cfgopt.getoption("--gdb", "") + should_gdb = gdbcmd and (shellopt == "all" or self.name in shellopt.split(",")) + use_emacs = self.unet.cfgopt.getoption("--gdb-use-emacs", False) + + if should_gdb and not use_emacs: + cmds = self.config.get("gdb-target-cmds", []) + for cmd in cmds: + gdbcmd += f" '-ex={cmd}'" + + bps = self.unet.cfgopt.getoption("--gdb-breakpoints", "").split(",") + for bp in bps: + gdbcmd += f" '-ex=b {bp}'" + + cmds = self.config.get("gdb-run-cmd", []) + for cmd in cmds: + gdbcmd += f" '-ex={cmd}'" + + self.run_in_window(gdbcmd) + elif should_gdb and use_emacs: + gdbcmd = gdbcmd.replace("gdb ", "gdb -i=mi ") + ecbin = self.get_exec_path("emacsclient") + # output = self.cmd_raises( + # [ecbin, "--eval", f"(gdb \"{gdbcmd} -ex='p 123456'\")"] + # ) + _ = self.cmd_raises([ecbin, "--eval", f'(gdb "{gdbcmd}")']) + + # can't figure out how to wait until symbols are loaded, until we do we just + # have to wait "long enough" for the symbol load to finish :/ + # for _ in range(100): + # output = self.cmd_raises( + # [ + # ecbin, + # "--eval", + # f"gdb-first-prompt", + # ] + # ) + # if output == "nil\n": + # break + # time.sleep(0.25) + + time.sleep(10) + + cmds = self.config.get("gdb-target-cmds", []) + for cmd in cmds: + # we may want to quote quotes in the cmd string + self.cmd_raises( + [ + ecbin, + "--eval", + f'(gud-gdb-run-command-fetch-lines "{cmd}" "*gud-gdb*")', + ] + ) + + bps = self.unet.cfgopt.getoption("--gdb-breakpoints", "").split(",") + for bp in bps: + cmd = f"br {bp}" + self.cmd_raises( + [ + ecbin, + "--eval", + f'(gud-gdb-run-command-fetch-lines "{cmd}" "*gud-gdb*")', + ] + ) + + cmds = self.config.get("gdb-run-cmds", []) + for cmd in cmds: + # we may want to quote quotes in the cmd string + self.cmd_raises( + [ + ecbin, + "--eval", + f'(gud-gdb-run-command-fetch-lines "{cmd}" "*gud-gdb*")', + ] + ) + gdbcmd += f" '-ex={cmd}'" + + shellopt = self.unet.cfgopt.getoption("--shell") + shellopt = shellopt if shellopt else "" + if shellopt == "all" or self.name in shellopt.split(","): + self.run_in_window("bash") + + async def _async_delete(self): + self.logger.debug("%s: NodeMixin sub-class _async_delete", self) + + if self.cmd_p: + await self.async_cleanup_proc(self.cmd_p, self.cmd_pid) + self.cmd_p = None + + # Next call users "cleanup_cmd:" + try: + if not self.cleanup_called: + await self.async_cleanup_cmd() + except Exception as error: + self.logger.warning( + "Got an error during delete from async_cleanup_cmd: %s", error + ) + + # delete the LinuxNamespace/InterfaceMixin + await super()._async_delete() + + +class SSHRemote(NodeMixin, Commander): + """SSHRemote a node representing an ssh connection to something.""" + + def __init__( + self, + name, + server, + port=22, + user=None, + password=None, + idfile=None, + **kwargs, + ): + super().__init__(name, **kwargs) + + self.logger.debug("%s: creating", self) + + # Things done in LinuxNamepsace we need to replicate here. + self.rundir = self.unet.rundir.joinpath(self.name) + self.unet.cmd_raises(f"rm -rf {self.rundir}") + self.unet.cmd_raises(f"mkdir -p {self.rundir}") + + self.mgmt_ip = None + self.mgmt_ip6 = None + + self.port = port + + if user: + self.user = user + elif "SUDO_USER" in os.environ: + self.user = os.environ["SUDO_USER"] + else: + self.user = getpass.getuser() + self.password = password + self.idfile = idfile + + self.server = f"{self.user}@{server}" + + # Setup our base `pre-cmd` values + # + # We maybe should add environment variable transfer here in particular + # MUNET_NODENAME. The problem is the user has to explicitly approve + # of SendEnv variables. + self.__base_cmd = [ + get_exec_path_host("sudo"), + "-E", + f"-u{self.user}", + get_exec_path_host("ssh"), + ] + if port != 22: + self.__base_cmd.append(f"-p{port}") + self.__base_cmd.append("-q") + self.__base_cmd.append("-oStrictHostKeyChecking=no") + self.__base_cmd.append("-oUserKnownHostsFile=/dev/null") + if self.idfile: + self.__base_cmd.append(f"-i{self.idfile}") + # Would be nice but has to be accepted by server config so not very useful. + # self.__base_cmd.append("-oSendVar='TEST'") + self.__base_cmd_pty = list(self.__base_cmd) + self.__base_cmd_pty.append("-t") + self.__base_cmd.append(self.server) + self.__base_cmd_pty.append(self.server) + # self.set_pre_cmd(pre_cmd, pre_cmd_tty) + + self.logger.info("%s: created", self) + + def has_ready_cmd(self) -> bool: + return bool(self.config.get("ready-cmd", "").strip()) + + def _get_pre_cmd(self, use_str, use_pty, ns_only=False, **kwargs): + pre_cmd = [] + if self.unet: + pre_cmd = self.unet._get_pre_cmd(False, use_pty, ns_only=False, **kwargs) + if ns_only: + return pre_cmd + + # XXX grab the env from kwargs and add to podman exec + # env = kwargs.get("env", {}) + if use_pty: + pre_cmd = pre_cmd + self.__base_cmd_pty + else: + pre_cmd = pre_cmd + self.__base_cmd + return shlex.join(pre_cmd) if use_str else list(pre_cmd) + + def _get_cmd_as_list(self, cmd): + """Given a list or string return a list form for execution. + + If cmd is a string then [cmd] is returned, for most other + node types ["bash", "-c", cmd] is returned but in our case + ssh is the shell. + + Args: + cmd: list or string representing the command to execute. + str_shell: if True and `cmd` is a string then run the + command using bash -c + Returns: + list of commands to execute. + """ + return [cmd] if isinstance(cmd, str) else cmd + + +# Would maybe like to refactor this into L3 and Node +class L3NodeMixin(NodeMixin): + """A linux namespace with IP attributes.""" + + def __init__(self, *args, unet=None, **kwargs): + """Create an L3Node.""" + # logging.warning( + # "L3NodeMixin: config %s unet %s kwargs %s", config, unet, kwargs + # ) + super().__init__(*args, unet=unet, **kwargs) + + self.mgmt_ip = None # set in parser.py + self.mgmt_ip6 = None # set in parser.py + self.host_intfs = {} + self.phy_intfs = {} + self.phycount = 0 + self.phy_odrivers = {} + self.tapmacs = {} + + self.intf_tc_count = 0 + + # super().__init__(name=name, **kwargs) + + self.mount_volumes() + + # ----------------------- + # Setup node's networking + # ----------------------- + if not unet.ipv6_enable: + # Disable IPv6 + self.cmd_raises("sysctl -w net.ipv6.conf.all.autoconf=0") + self.cmd_raises("sysctl -w net.ipv6.conf.all.disable_ipv6=1") + else: + self.cmd_raises("sysctl -w net.ipv6.conf.all.autoconf=1") + self.cmd_raises("sysctl -w net.ipv6.conf.all.disable_ipv6=0") + + self.next_p2p_network = ipaddress.ip_network(f"10.254.{self.id}.0/31") + self.next_p2p_network6 = ipaddress.ip_network(f"fcff:ffff:{self.id:02x}::/127") + + self.loopback_ip = None + self.loopback_ips = get_loopback_ips(self.config, self.id) + self.loopback_ip = self.loopback_ips[0] if self.loopback_ips else None + if self.loopback_ip: + self.cmd_raises_nsonly(f"ip addr add {self.loopback_ip} dev lo") + self.cmd_raises_nsonly("ip link set lo up") + for i, ip in enumerate(self.loopback_ips[1:]): + self.cmd_raises_nsonly(f"ip addr add {ip} dev lo:{i}") + + # ------------------- + # Setup node's rundir + # ------------------- + + # Not host path based, but we assume same + self.set_ns_cwd(self.rundir) + + # Save the namespace pid + with open(os.path.join(self.rundir, "nspid"), "w", encoding="ascii") as f: + f.write(f"{self.pid}\n") + + with open(os.path.join(self.rundir, "nspids"), "w", encoding="ascii") as f: + f.write(f'{" ".join([str(x) for x in self.pids])}\n') + + # Create a hosts file to map our name + hosts_file = os.path.join(self.rundir, "hosts.txt") + with open(hosts_file, "w", encoding="ascii") as hf: + hf.write( + f"""127.0.0.1\tlocalhost {self.name} +::1\tip6-localhost ip6-loopback +fe00::0\tip6-localnet +ff00::0\tip6-mcastprefix +ff02::1\tip6-allnodes +ff02::2\tip6-allrouters +""" + ) + if hasattr(self, "bind_mount"): + self.bind_mount(hosts_file, "/etc/hosts") + + async def console( + self, + concmd, + prompt=r"(^|\r?\n)[^#\$]*[#\$] ", + is_bourne=True, + user=None, + password=None, + expects=None, + sends=None, + use_pty=False, + will_echo=False, + logfile_prefix="console", + trace=True, + **kwargs, + ): + """Create a REPL (read-eval-print-loop) driving a console. + + Args: + concmd: string or list to popen with, or an already open socket + prompt: the REPL prompt to look for, the function returns when seen + is_bourne: True if the console is a bourne shell + user: user name to log in with + password: password to log in with + expects: a list of regex other than the prompt, the standard user, or + password to look for. "ogin:" or "[Pp]assword:"r. + sends: what to send when an element of `expects` matches. Can be the + empty string to send nothing. + use_pty: true for pty based expect, otherwise uses popen (pipes/files) + will_echo: bash is buggy in that it echo's to non-tty unlike any other + sh/ksh, set this value to true if running back + logfile_prefix: prefix for 3 logfiles opened to track the console i/o + trace: trace the send/expect sequence + **kwargs: kwargs passed on the _spawn. + """ + lfname = os.path.join(self.rundir, f"{logfile_prefix}-log.txt") + logfile = open(lfname, "a+", encoding="utf-8") + logfile.write("-- start logging for: '{}' --\n".format(concmd)) + + lfname = os.path.join(self.rundir, f"{logfile_prefix}-read-log.txt") + logfile_read = open(lfname, "a+", encoding="utf-8") + logfile_read.write("-- start read logging for: '{}' --\n".format(concmd)) + + lfname = os.path.join(self.rundir, f"{logfile_prefix}-send-log.txt") + logfile_send = open(lfname, "a+", encoding="utf-8") + logfile_send.write("-- start send logging for: '{}' --\n".format(concmd)) + + expects = [] if expects is None else expects + sends = [] if sends is None else sends + if user: + expects.append("ogin:") + sends.append(user + "\n") + if password is not None: + expects.append("assword:") + sends.append(password + "\n") + repl = await self.shell_spawn( + concmd, + prompt, + expects=expects, + sends=sends, + use_pty=use_pty, + will_echo=will_echo, + is_bourne=is_bourne, + logfile=logfile, + logfile_read=logfile_read, + logfile_send=logfile_send, + trace=trace, + **kwargs, + ) + return repl + + async def monitor( + self, + sockpath, + prompt=r"\(qemu\) ", + ): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(sockpath) + + pfx = os.path.basename(sockpath) + + lfname = os.path.join(self.rundir, f"{pfx}-log.txt") + logfile = open(lfname, "a+", encoding="utf-8") + logfile.write("-- start logging for: '{}' --\n".format(sock)) + + lfname = os.path.join(self.rundir, f"{pfx}-read-log.txt") + logfile_read = open(lfname, "a+", encoding="utf-8") + logfile_read.write("-- start read logging for: '{}' --\n".format(sock)) + + p = self.spawn(sock, prompt, logfile=logfile, logfile_read=logfile_read) + from .base import ShellWrapper # pylint: disable=C0415 + + p.send("\n") + return ShellWrapper(p, prompt, None, will_echo=True, escape_ansi=True) + + def mount_volumes(self): + for m in self.config.get("volumes", []): + if isinstance(m, str): + s = m.split(":", 1) + if len(s) == 1: + self.tmpfs_mount(s[0]) + else: + spath = s[0] + if spath[0] == ".": + spath = os.path.abspath( + os.path.join(self.unet.config_dirname, spath) + ) + self.bind_mount(spath, s[1]) + continue + raise NotImplementedError("complex mounts for non-containers") + + def get_ifname(self, netname): + for c in self.config["connections"]: + if c["to"] == netname: + return c["name"] + return None + + def set_lan_addr(self, switch, cconf): + if ip := cconf.get("ip"): + ipaddr = ipaddress.ip_interface(ip) + assert ipaddr.version == 4 + elif self.unet.autonumber and "ip" not in cconf: + self.logger.debug( + "%s: prefixlen of switch %s is %s", + self, + switch.name, + switch.ip_network.prefixlen, + ) + n = switch.ip_network + ipaddr = ipaddress.ip_interface((n.network_address + self.id, n.prefixlen)) + else: + ipaddr = None + + if ip := cconf.get("ipv6"): + ip6addr = ipaddress.ip_interface(ip) + assert ipaddr.version == 6 + elif self.unet.ipv6_enable and self.unet.autonumber and "ipv6" not in cconf: + self.logger.debug( + "%s: prefixlen of switch %s is %s", + self, + switch.name, + switch.ip6_network.prefixlen, + ) + n = switch.ip6_network + ip6addr = ipaddress.ip_interface((n.network_address + self.id, n.prefixlen)) + else: + ip6addr = None + + dns_network = self.unet.topoconf.get("dns-network") + for ip in (ipaddr, ip6addr): + if not ip: + continue + ipcmd = "ip " if ip.version == 4 else "ip -6 " + if dns_network and dns_network == switch.name: + if ip.version == 4: + self.mgmt_ip = ip.ip + else: + self.mgmt_ip6 = ip.ip + ifname = cconf["name"] + self.set_intf_addr(ifname, ip) + self.logger.debug("%s: adding %s to lan intf %s", self, ip, ifname) + if not self.is_vm: + self.intf_ip_cmd(ifname, ipcmd + f"addr add {ip} dev {ifname}") + if hasattr(switch, "is_nat") and switch.is_nat: + swaddr = ( + switch.ip_address if ip.version == 4 else switch.ip6_address + ) + self.cmd_raises(ipcmd + f"route add default via {swaddr}") + + def _set_p2p_addr(self, other, cconf, occonf, ipv6=False): + ipkey = "ipv6" if ipv6 else "ip" + ipaddr = ipaddress.ip_interface(cconf[ipkey]) if cconf.get(ipkey) else None + oipaddr = ipaddress.ip_interface(occonf[ipkey]) if occonf.get(ipkey) else None + self.logger.debug( + "%s: set_p2p_addr %s %s %s", self, other.name, ipaddr, oipaddr + ) + + if not ipaddr and not oipaddr: + if self.unet.autonumber: + if ipv6: + n = self.next_p2p_network6 + self.next_p2p_network6 = make_ip_network(n, 1) + else: + n = self.next_p2p_network + self.next_p2p_network = make_ip_network(n, 1) + + ipaddr = ipaddress.ip_interface(n) + oipaddr = ipaddress.ip_interface((ipaddr.ip + 1, n.prefixlen)) + else: + return + + if ipaddr: + ifname = cconf["name"] + self.set_intf_addr(ifname, ipaddr) + self.logger.debug("%s: adding %s to p2p intf %s", self, ipaddr, ifname) + if "physical" not in cconf and not self.is_vm: + self.intf_ip_cmd(ifname, f"ip addr add {ipaddr} dev {ifname}") + + if oipaddr: + oifname = occonf["name"] + other.set_intf_addr(oifname, oipaddr) + self.logger.debug( + "%s: adding %s to other p2p intf %s", other, oipaddr, oifname + ) + if "physical" not in occonf and not other.is_vm: + other.intf_ip_cmd(oifname, f"ip addr add {oipaddr} dev {oifname}") + + def set_p2p_addr(self, other, cconf, occonf): + self._set_p2p_addr(other, cconf, occonf, ipv6=False) + if self.unet.ipv6_enable: + self._set_p2p_addr(other, cconf, occonf, ipv6=True) + + async def add_host_intf(self, hname, lname, mtu=None): + if hname in self.host_intfs: + return + self.host_intfs[hname] = lname + self.unet.rootcmd.cmd_nostatus(f"ip link set {hname} down ") + self.unet.rootcmd.cmd_raises(f"ip link set {hname} netns {self.pid}") + self.cmd_raises(f"ip link set {hname} name {lname}") + if mtu: + self.cmd_raises(f"ip link set {lname} mtu {mtu}") + self.cmd_raises(f"ip link set {lname} up") + + async def rem_host_intf(self, hname): + lname = self.host_intfs[hname] + self.cmd_raises(f"ip link set {lname} down") + self.cmd_raises(f"ip link set {lname} name {hname}") + self.cmd_raises(f"ip link set {hname} netns 1") + del self.host_intfs[hname] + + async def add_phy_intf(self, devaddr, lname): + """Add a physical inteface (i.e. mv it to vfio-pci driver. + + This is primarily useful for Qemu, but also for things like TREX or DPDK + """ + if devaddr in self.phy_intfs: + return + self.phy_intfs[devaddr] = lname + index = len(self.phy_intfs) + + _, _, off, fun = parse_pciaddr(devaddr) + doffset = off * 8 + fun + + is_virtual = self.unet.rootcmd.path_exists( + f"/sys/bus/pci/devices/{devaddr}/physfn" + ) + if is_virtual: + pfname = self.unet.rootcmd.cmd_raises( + f"ls -1 /sys/bus/pci/devices/{devaddr}/physfn/net" + ).strip() + pdevaddr = read_sym_basename(f"/sys/bus/pci/devices/{devaddr}/physfn") + _, _, poff, pfun = parse_pciaddr(pdevaddr) + poffset = poff * 8 + pfun + + offset = read_int_value( + f"/sys/bus/pci/devices/{devaddr}/physfn/sriov_offset" + ) + stride = read_int_value( + f"/sys/bus/pci/devices/{devaddr}/physfn/sriov_stride" + ) + vf = (doffset - offset - poffset) // stride + mac = f"02:cc:cc:cc:{index:02x}:{self.id:02x}" + # Some devices require the parent to be up (e.g., ixbge) + self.unet.rootcmd.cmd_raises(f"ip link set {pfname} up") + self.unet.rootcmd.cmd_raises(f"ip link set {pfname} vf {vf} mac {mac}") + self.unet.rootcmd.cmd_status(f"ip link set {pfname} vf {vf} trust on") + self.tapmacs[devaddr] = mac + + self.logger.info("Adding physical PCI device %s as %s", devaddr, lname) + + # Get interface name and set to down if present + ec, ifname, _ = self.unet.rootcmd.cmd_status( + f"ls /sys/bus/pci/devices/{devaddr}/net/", warn=False + ) + ifname = ifname.strip() + if not ec and ifname: + # XXX Should only do this is the device is up, and then likewise return it + # up on exit self.phy_intfs_hostname[devaddr] = ifname + self.logger.info( + "Setting physical PCI device %s named %s down", devaddr, ifname + ) + self.unet.rootcmd.cmd_status( + f"ip link set {ifname} down 2> /dev/null || true" + ) + + # Get the current bound driver, and unbind + try: + driver = read_sym_basename(f"/sys/bus/pci/devices/{devaddr}/driver") + driver = driver.strip() + except Exception: + driver = "" + if driver: + if driver == "vfio-pci": + self.logger.info( + "Physical PCI device %s already bound to vfio-pci", devaddr + ) + return + self.logger.info( + "Unbinding physical PCI device %s from driver %s", devaddr, driver + ) + self.phy_odrivers[devaddr] = driver + self.unet.rootcmd.cmd_raises( + f"echo {devaddr} > /sys/bus/pci/drivers/{driver}/unbind" + ) + + # Add the device vendor and device id to vfio-pci in case it's the first time + vendor = read_str_value(f"/sys/bus/pci/devices/{devaddr}/vendor") + devid = read_str_value(f"/sys/bus/pci/devices/{devaddr}/device") + self.logger.info("Adding device IDs %s:%s to vfio-pci", vendor, devid) + ec, _, _ = self.unet.rootcmd.cmd_status( + f"echo {vendor} {devid} > /sys/bus/pci/drivers/vfio-pci/new_id", warn=False + ) + + if not self.unet.rootcmd.path_exists(f"/sys/bus/pci/driver/vfio-pci/{devaddr}"): + # Bind to vfio-pci if wasn't added with new_id + self.logger.info("Binding physical PCI device %s to vfio-pci", devaddr) + ec, _, _ = self.unet.rootcmd.cmd_status( + f"echo {devaddr} > /sys/bus/pci/drivers/vfio-pci/bind" + ) + + async def rem_phy_intf(self, devaddr): + """Remove a physical inteface (i.e. mv it away from vfio-pci driver. + + This is primarily useful for Qemu, but also for things like TREX or DPDK + """ + lname = self.phy_intfs.get(devaddr, "") + if lname: + del self.phy_intfs[devaddr] + + # ifname = self.phy_intfs_hostname.get(devaddr, "") + # if ifname + # del self.phy_intfs_hostname[devaddr] + + driver = self.phy_odrivers.get(devaddr, "") + if not driver: + self.logger.info( + "Physical PCI device %s was bound to vfio-pci on entry", devaddr + ) + return + + self.logger.info( + "Unbinding physical PCI device %s from driver vfio-pci", devaddr + ) + self.unet.rootcmd.cmd_status( + f"echo {devaddr} > /sys/bus/pci/drivers/vfio-pci/unbind" + ) + + self.logger.info("Binding physical PCI device %s to driver %s", devaddr, driver) + ec, _, _ = self.unet.rootcmd.cmd_status( + f"echo {devaddr} > /sys/bus/pci/drivers/{driver}/bind" + ) + if not ec: + del self.phy_odrivers[devaddr] + + async def _async_delete(self): + self.logger.debug("%s: L3NodeMixin sub-class _async_delete", self) + + # XXX do we need to run the cleanup command before these infra changes? + + # remove any hostintf interfaces + for hname in list(self.host_intfs): + await self.rem_host_intf(hname) + + # remove any hostintf interfaces + for devaddr in list(self.phy_intfs): + await self.rem_phy_intf(devaddr) + + # delete the LinuxNamespace/InterfaceMixin + await super()._async_delete() + + +class L3NamespaceNode(L3NodeMixin, LinuxNamespace): + """A namespace L3 node.""" + + def __init__(self, name, pid=True, **kwargs): + # logging.warning( + # "L3NamespaceNode: name %s MRO: %s kwargs %s", + # name, + # L3NamespaceNode.mro(), + # kwargs, + # ) + super().__init__(name, pid=pid, **kwargs) + super().pytest_hook_open_shell() + + async def _async_delete(self): + self.logger.debug("%s: deleting", self) + await super()._async_delete() + + +class L3ContainerNode(L3NodeMixin, LinuxNamespace): + """An container (podman) based L3 node.""" + + def __init__(self, name, config, **kwargs): + """Create a Container Node.""" + self.cont_exec_paths = {} + self.container_id = None + self.container_image = config["image"] + self.extra_mounts = [] + assert self.container_image + + self.cmd_p = None + self.__base_cmd = [] + self.__base_cmd_pty = [] + + # don't we have a mutini or cat process? + super().__init__( + name=name, + config=config, + # pid=True, + # cgroup=True, + # private_mounts=["/sys/fs/cgroup:/sys/fs/cgroup"], + **kwargs, + ) + + @property + def is_container(self): + return True + + def get_exec_path(self, binary): + """Return the full path to the binary executable inside the image. + + `binary` :: binary name or list of binary names + """ + return _get_exec_path(binary, self.cmd_status, self.cont_exec_paths) + + async def async_get_exec_path(self, binary): + """Return the full path to the binary executable inside the image. + + `binary` :: binary name or list of binary names + """ + path = await _async_get_exec_path( + binary, self.async_cmd_status, self.cont_exec_paths + ) + return path + + def get_exec_path_host(self, binary): + """Return the full path to the binary executable on the host. + + `binary` :: binary name or list of binary names + """ + return get_exec_path_host(binary) + + def _get_pre_cmd(self, use_str, use_pty, ns_only=False, root_level=False, **kwargs): + if ns_only: + return super()._get_pre_cmd( + use_str, use_pty, ns_only=True, root_level=root_level, **kwargs + ) + if not self.cmd_p: + if self.container_id: + s = f"{self}: Running command in namespace b/c container exited" + self.logger.warning("%s", s) + raise L3ContainerNotRunningError(s) + self.logger.debug("%s: Running command in namespace b/c no container", self) + return super()._get_pre_cmd( + use_str, use_pty, ns_only=True, root_level=root_level, **kwargs + ) + + # We need to enter our namespaces when running the podman command + pre_cmd = super()._get_pre_cmd( + False, use_pty, ns_only=True, root_level=root_level, **kwargs + ) + + # XXX grab the env from kwargs and add to podman exec + # env = kwargs.get("env", {}) + if use_pty: + pre_cmd = pre_cmd + self.__base_cmd_pty + else: + pre_cmd = pre_cmd + self.__base_cmd + return shlex.join(pre_cmd) if use_str else pre_cmd + + def tmpfs_mount(self, inner): + # eventually would be nice to support live mounting + assert not self.container_id + self.logger.debug("Mounting tmpfs on %s", inner) + self.extra_mounts.append(f"--mount=type=tmpfs,destination={inner}") + + def bind_mount(self, outer, inner): + # eventually would be nice to support live mounting + assert not self.container_id + # First bind the mount in the parent this allows things like /etc/hosts to work + # correctly when running "nsonly" commands + super().bind_mount(outer, inner) + # Then arrange for binding in the container as well. + self.logger.debug("Bind mounting %s on %s", outer, inner) + if not self.test_nsonly("-e", outer): + self.cmd_raises_nsonly(f"mkdir -p {outer}") + self.extra_mounts.append(f"--mount=type=bind,src={outer},dst={inner}") + + def mount_volumes(self): + args = [] + for m in self.config.get("volumes", []): + if isinstance(m, str): + s = m.split(":", 1) + if len(s) == 1: + args.append("--mount=type=tmpfs,destination=" + m) + else: + spath = s[0] + spath = os.path.abspath( + os.path.join( + os.path.dirname(self.unet.config["config_pathname"]), spath + ) + ) + if not self.test_nsonly("-e", spath): + self.cmd_raises_nsonly(f"mkdir -p {spath}") + args.append(f"--mount=type=bind,src={spath},dst={s[1]}") + continue + + for m in self.config.get("mounts", []): + margs = ["type=" + m["type"]] + for k, v in m.items(): + if k == "type": + continue + if v: + if k in ("src", "source"): + v = os.path.abspath( + os.path.join( + os.path.dirname(self.unet.config["config_pathname"]), v + ) + ) + if not self.test_nsonly("-e", v): + self.cmd_raises_nsonly(f"mkdir -p {v}") + margs.append(f"{k}={v}") + else: + margs.append(f"{k}") + args.append("--mount=" + ",".join(margs)) + + if args: + # Need to work on a way to mount into live container too + self.extra_mounts += args + + def has_run_cmd(self) -> bool: + return True + + async def run_cmd(self): + """Run the configured commands for this node.""" + self.logger.debug("%s: starting container", self.name) + self.logger.debug( + "[rundir %s exists %s]", self.rundir, os.path.exists(self.rundir) + ) + + self.container_id = f"{self.name}-{os.getpid()}" + proc_path = self.unet.proc_path if self.unet else "/proc" + cmds = [ + get_exec_path_host("podman"), + "run", + f"--name={self.container_id}", + # f"--net=ns:/proc/{self.pid}/ns/net", + f"--net=ns:{proc_path}/{self.pid}/ns/net", + f"--hostname={self.name}", + f"--add-host={self.name}:127.0.0.1", + # We can't use --rm here b/c podman fails on "stop". + # u"--rm", + ] + + if self.config.get("init", True): + cmds.append("--init") + + if self.config.get("privileged", False): + cmds.append("--privileged") + # If we don't do this then the host file system is remounted read-only on + # exit! + cmds.append("--systemd=false") + else: + cmds.extend( + [ + # "--cap-add=SYS_ADMIN", + "--cap-add=NET_ADMIN", + "--cap-add=NET_RAW", + ] + ) + + # Add volumes: + if self.extra_mounts: + cmds += self.extra_mounts + + # Add environment variables: + envdict = self.config.get("env", {}) + if envdict is None: + envdict = {} + for k, v in envdict.items(): + cmds.append(f"--env={k}={v}") + + # Update capabilities + cmds += [f"--cap-add={x}" for x in self.config.get("cap-add", [])] + cmds += [f"--cap-drop={x}" for x in self.config.get("cap-drop", [])] + # cmds += [f"--expose={x.split(':')[0]}" for x in self.config.get("ports", [])] + cmds += [f"--publish={x}" for x in self.config.get("ports", [])] + + # Add extra flags from user: + if "podman" in self.config: + for x in self.config["podman"].get("extra-args", []): + cmds.append(x.strip()) + + # shell_cmd is a union and can be boolean or string + shell_cmd = self.config.get("shell", "/bin/bash") + if not isinstance(shell_cmd, str): + if shell_cmd: + shell_cmd = "/bin/bash" + else: + shell_cmd = "" + + # Create shebang files, filled later on + for key in ("cleanup-cmd", "ready-cmd"): + shebang_cmd = self.config.get(key, "").strip() + if shell_cmd and shebang_cmd: + script_name = fsafe_name(key) + # Will write the file contents out when the command is run + shebang_cmdpath = os.path.join(self.rundir, f"{script_name}.shebang") + await self.async_cmd_raises_nsonly(f"touch {shebang_cmdpath}") + await self.async_cmd_raises_nsonly(f"chmod 755 {shebang_cmdpath}") + cmds += [ + # How can we override this? + # u'--entrypoint=""', + f"--volume={shebang_cmdpath}:/tmp/{script_name}.shebang", + ] + + cmd = self.config.get("cmd", "").strip() + + # See if we have a custom update for this `kind` + if kind := self.config.get("kind", None): + if kind in kind_run_cmd_update: + cmds, cmd = await kind_run_cmd_update[kind](self, shell_cmd, cmds, cmd) + + # Create running command file + if shell_cmd and cmd: + assert isinstance(cmd, str) + # make cmd \n terminated for script + cmd = cmd.rstrip() + cmd = cmd.replace("%CONFIGDIR%", str(self.unet.config_dirname)) + cmd = cmd.replace("%RUNDIR%", str(self.rundir)) + cmd = cmd.replace("%NAME%", str(self.name)) + cmd += "\n" + cmdpath = os.path.join(self.rundir, "cmd.shebang") + with open(cmdpath, mode="w+", encoding="utf-8") as cmdfile: + cmdfile.write(f"#!{shell_cmd}\n") + cmdfile.write(cmd) + cmdfile.flush() + self.cmd_raises_nsonly(f"chmod 755 {cmdpath}") + cmds += [ + # How can we override this? + # u'--entrypoint=""', + f"--volume={cmdpath}:/tmp/cmds.shebang", + self.container_image, + "/tmp/cmds.shebang", + ] + else: + # `cmd` is a direct run (no shell) cmd + cmds.append(self.container_image) + if cmd: + if isinstance(cmd, str): + cmds.extend(shlex.split(cmd)) + else: + cmds.extend(cmd) + + cmds = [ + x.replace("%CONFIGDIR%", str(self.unet.config_dirname)) for x in cmds + ] + cmds = [x.replace("%RUNDIR%", str(self.rundir)) for x in cmds] + cmds = [x.replace("%NAME%", str(self.name)) for x in cmds] + + stdout = open(os.path.join(self.rundir, "cmd.out"), "wb") + stderr = open(os.path.join(self.rundir, "cmd.err"), "wb") + # Using nsonly avoids using `podman exec` to execute the cmds. + self.cmd_p = await self.async_popen_nsonly( + cmds, + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + start_new_session=True, # keeps main tty signals away from podman + ) + + self.logger.debug("%s: async_popen => %s", self, self.cmd_p.pid) + + self.pytest_hook_run_cmd(stdout, stderr) + + # --------------------------------------- + # Now let's wait until container shows up + # --------------------------------------- + timeout = Timeout(30) + while self.cmd_p.returncode is None and not timeout.is_expired(): + o = await self.async_cmd_raises_nsonly( + f"podman ps -q -f name={self.container_id}" + ) + if o.strip(): + break + elapsed = int(timeout.elapsed()) + if elapsed <= 3: + await asyncio.sleep(0.1) + else: + self.logger.info("%s: run_cmd taking more than %ss", self, elapsed) + await asyncio.sleep(1) + if self.cmd_p.returncode is not None: + # leave self.container_id set to cause exception on use + self.logger.warning( + "%s: run_cmd exited quickly (%ss) rc: %s", + self, + timeout.elapsed(), + self.cmd_p.returncode, + ) + elif timeout.is_expired(): + self.logger.critical( + "%s: timeout (%ss) waiting for container to start", + self.name, + timeout.elapsed(), + ) + assert not timeout.is_expired() + + # + # Set our precmd for executing in the container + # + self.__base_cmd = [ + get_exec_path_host("podman"), + "exec", + f"-eMUNET_RUNDIR={self.unet.rundir}", + f"-eMUNET_NODENAME={self.name}", + "-i", + ] + self.__base_cmd_pty = list(self.__base_cmd) # copy list to pty + self.__base_cmd.append(self.container_id) # end regular list + self.__base_cmd_pty.append("-t") # add pty flags + self.__base_cmd_pty.append(self.container_id) # end pty list + # self.set_pre_cmd(self.__base_cmd, self.__base_cmd_pty) # set both pre_cmd + + self.logger.info("%s: started container", self.name) + + self.pytest_hook_open_shell() + + return self.cmd_p + + async def async_cleanup_cmd(self): + """Run the configured cleanup commands for this node.""" + self.cleanup_called = True + + if "cleanup-cmd" not in self.config: + return + + if not self.cmd_p: + self.logger.warning("async_cleanup_cmd: container no longer running") + return + + return await self._async_cleanup_cmd() + + def cmd_completed(self, future): + try: + log = self.logger.debug if self.deleting else self.logger.warning + n = future.result() + if self.deleting: + log("contianer `cmd:` result: %s", n) + else: + log( + "contianer `cmd:` exited early, " + "try adding `tail -f /dev/null` to `cmd:`, result: %s", + n, + ) + except asyncio.CancelledError as error: + # Should we stop the container if we have one? or since we are canceled + # we know we will be deleting soon? + self.logger.warning( + "node container cmd wait() canceled: %s:%s", future, error + ) + self.cmd_p = None + + async def _async_delete(self): + self.logger.debug("%s: deleting", self) + + if contid := self.container_id: + try: + if not self.cleanup_called: + self.logger.debug("calling user cleanup cmd") + await self.async_cleanup_cmd() + except Exception as error: + self.logger.warning( + "Got an error during delete from async_cleanup_cmd: %s", error + ) + + # Clear the container_id field we want to act like a namespace now. + self.container_id = None + + o = "" + e = "" + if self.cmd_p: + self.logger.debug("podman stop on container: %s", contid) + if (rc := self.cmd_p.returncode) is None: + rc, o, e = await self.async_cmd_status_nsonly( + [get_exec_path_host("podman"), "stop", "--time=2", contid] + ) + if rc and rc < 128: + self.logger.warning( + "%s: podman stop on cmd failed: %s", + self, + cmd_error(rc, o, e), + ) + else: + # It's gone + self.cmd_p = None + + # now remove the container + self.logger.debug("podman rm on container: %s", contid) + rc, o, e = await self.async_cmd_status_nsonly( + [get_exec_path_host("podman"), "rm", contid] + ) + if rc: + self.logger.warning( + "%s: podman rm failed: %s", self, cmd_error(rc, o, e) + ) + else: + self.logger.debug( + "podman removed container %s: %s", contid, cmd_error(rc, o, e) + ) + + await super()._async_delete() + + +class L3QemuVM(L3NodeMixin, LinuxNamespace): + """An VM (qemu) based L3 node.""" + + def __init__(self, name, config, **kwargs): + """Create a Container Node.""" + self.cont_exec_paths = {} + self.launch_p = None + self.qemu_config = config["qemu"] + self.extra_mounts = [] + assert self.qemu_config + self.cmdrepl = None + self.conrepl = None + self.is_kvm = False + self.monrepl = None + self.tapfds = {} + self.cpu_thread_map = {} + + self.tapnames = {} + + self.use_ssh = False + self.__base_cmd = [] + self.__base_cmd_pty = [] + + super().__init__(name=name, config=config, pid=False, **kwargs) + + self.sockdir = self.rundir.joinpath("s") + self.cmd_raises(f"mkdir -p {self.sockdir}") + + self.qemu_config = config_subst( + self.qemu_config, + name=self.name, + rundir=os.path.join(self.rundir, self.name), + configdir=self.unet.config_dirname, + ) + self.ssh_keyfile = self.qemu_config.get("sshkey") + + @property + def is_vm(self): + return True + + def __setup_ssh(self): + if not self.ssh_keyfile: + self.logger.warning("%s: No sshkey config", self) + return False + if not self.mgmt_ip and not self.mgmt_ip6: + self.logger.warning("%s: No mgmt IP to ssh to", self) + return False + mgmt_ip = self.mgmt_ip if self.mgmt_ip else self.mgmt_ip6 + + # + # Since we have a keyfile shouldn't need to sudo + # self.user = os.environ.get("SUDO_USER", "") + # if not self.user: + # self.user = getpass.getuser() + # self.__base_cmd = [ + # get_exec_path_host("sudo"), + # "-E", + # f"-u{self.user}", + # get_exec_path_host("ssh"), + # ] + # + port = 22 + self.__base_cmd = [get_exec_path_host("ssh")] + if port != 22: + self.__base_cmd.append(f"-p{port}") + self.__base_cmd.append("-i") + self.__base_cmd.append(self.ssh_keyfile) + self.__base_cmd.append("-q") + self.__base_cmd.append("-oStrictHostKeyChecking=no") + self.__base_cmd.append("-oUserKnownHostsFile=/dev/null") + # Would be nice but has to be accepted by server config so not very useful. + # self.__base_cmd.append("-oSendVar='TEST'") + self.__base_cmd_pty = list(self.__base_cmd) + self.__base_cmd_pty.append("-t") + + user = self.qemu_config.get("sshuser", "root") + self.__base_cmd.append(f"{user}@{mgmt_ip}") + self.__base_cmd.append("--") + self.__base_cmd_pty.append(f"{user}@{mgmt_ip}") + # self.__base_cmd_pty.append("--") + return True + + def _get_cmd_as_list(self, cmd): + """Given a list or string return a list form for execution. + + If cmd is a string then [cmd] is returned, for most other + node types ["bash", "-c", cmd] is returned but in our case + ssh is the shell. + + Args: + cmd: list or string representing the command to execute. + str_shell: if True and `cmd` is a string then run the + command using bash -c + Returns: + list of commands to execute. + """ + if self.use_ssh and self.launch_p: + return [cmd] if isinstance(cmd, str) else cmd + return super()._get_cmd_as_list(cmd) + + def _get_pre_cmd(self, use_str, use_pty, ns_only=False, root_level=False, **kwargs): + if ns_only: + return super()._get_pre_cmd( + use_str, use_pty, ns_only=True, root_level=root_level, **kwargs + ) + + if not self.launch_p: + self.logger.debug("%s: Running command in namespace b/c no VM", self) + return super()._get_pre_cmd( + use_str, use_pty, ns_only=True, root_level=root_level, **kwargs + ) + + if not self.use_ssh: + self.logger.debug( + "%s: Running command in namespace b/c no SSH configured", self + ) + return super()._get_pre_cmd( + use_str, use_pty, ns_only=True, root_level=root_level, **kwargs + ) + + pre_cmd = self.unet._get_pre_cmd(use_str, use_pty, ns_only=True) + + # This is going to run in the process namespaces. + # We really want it to run in the munet namespace which will + # be different unless unshare_inline was used. + # + # XXX grab the env from kwargs and add to podman exec + # env = kwargs.get("env", {}) + if use_pty: + pre_cmd = pre_cmd + self.__base_cmd_pty + else: + pre_cmd = pre_cmd + self.__base_cmd + return shlex.join(pre_cmd) if use_str else pre_cmd + + async def moncmd(self): + """Uses internal REPL to send cmmand to qemu monitor and get reply.""" + + def tmpfs_mount(self, inner): + # eventually would be nice to support live mounting + self.logger.debug("Mounting tmpfs on %s", inner) + self.extra_mounts.append(("", inner, "")) + + # + # bind_mount is actually being used to mount into the namespace + # + # def bind_mount(self, outer, inner): + # # eventually would be nice to support live mounting + # assert not self.container_id + # if self.test_host("-f", outer): + # self.logger.warning("Can't bind mount files with L3QemuVM: %s", outer) + # return + # self.logger.debug("Bind mounting %s on %s", outer, inner) + # if not self.test_host("-e", outer): + # self.cmd_raises(f"mkdir -p {outer}") + # self.extra_mounts.append((outer, inner, "")) + + def mount_volumes(self): + """Mount volumes from the config.""" + args = [] + for m in self.config.get("volumes", []): + if not isinstance(m, str): + continue + s = m.split(":", 1) + if len(s) == 1: + args.append(("", s[0], "")) + else: + spath = s[0] + spath = os.path.abspath( + os.path.join( + os.path.dirname(self.unet.config["config_pathname"]), spath + ) + ) + if not self.test_nsonly("-e", spath): + self.cmd_raises_nsonly(f"mkdir -p {spath}") + args.append((spath, s[1], "")) + + for m in self.config.get("mounts", []): + src = m.get("src", m.get("source", "")) + if src: + src = os.path.abspath( + os.path.join( + os.path.dirname(self.unet.config["config_pathname"]), src + ) + ) + if not self.test_nsonly("-e", src): + self.cmd_raises_nsonly(f"mkdir -p {src}") + dst = m.get("dst", m.get("destination")) + assert dst, "destination path required for mount" + + margs = [] + for k, v in m.items(): + if k in ["destination", "dst", "source", "src"]: + continue + if k == "type": + assert v in ["bind", "tmpfs"] + continue + if not v: + margs.append(k) + else: + margs.append(f"{k}={v}") + args.append((src, dst, ",".join(margs))) + + if args: + self.extra_mounts += args + + async def run_cmd(self): + """Run the configured commands for this node inside VM.""" + self.logger.debug( + "[rundir %s exists %s]", self.rundir, os.path.exists(self.rundir) + ) + + cmd = self.config.get("cmd", "").strip() + if not cmd: + self.logger.debug("%s: no `cmd` to run", self) + return None + + shell_cmd = self.config.get("shell", "/bin/bash") + if not isinstance(shell_cmd, str): + if shell_cmd: + shell_cmd = "/bin/bash" + else: + shell_cmd = "" + + if shell_cmd: + cmd = cmd.rstrip() + cmd = f"#!{shell_cmd}\n" + cmd + cmd = cmd.replace("%CONFIGDIR%", str(self.unet.config_dirname)) + cmd = cmd.replace("%RUNDIR%", str(self.rundir)) + cmd = cmd.replace("%NAME%", str(self.name)) + cmd += "\n" + + # Write a copy to the rundir + cmdpath = os.path.join(self.rundir, "cmd.shebang") + with open(cmdpath, mode="w+", encoding="utf-8") as cmdfile: + cmdfile.write(cmd) + commander.cmd_raises(f"chmod 755 {cmdpath}") + + # Now write a copy inside the VM + self.conrepl.cmd_status("cat > /tmp/cmd.shebang << EOF\n" + cmd + "\nEOF") + self.conrepl.cmd_status("chmod 755 /tmp/cmd.shebang") + cmds = "/tmp/cmd.shebang" + else: + cmd = cmd.replace("%CONFIGDIR%", str(self.unet.config_dirname)) + cmd = cmd.replace("%RUNDIR%", str(self.rundir)) + cmd = cmd.replace("%NAME%", str(self.name)) + cmds = cmd + + # class future_proc: + # """Treat awaitable minimally as a proc.""" + # def __init__(self, aw): + # self.aw = aw + # # XXX would be nice to have a real value here + # self.returncode = 0 + # async def wait(self): + # if self.aw: + # return await self.aw + # return None + + class now_proc: + """Treat awaitable minimally as a proc.""" + + def __init__(self, output): + self.output = output + self.returncode = 0 + + async def wait(self): + return self.output + + if self.cmdrepl: + # self.cmd_p = future_proc( + # # We need our own console here b/c this is async and not returning + # # immediately + # # self.cmdrepl.run_command(cmds, timeout=120, async_=True) + # self.cmdrepl.run_command(cmds, timeout=120) + # ) + + # When run_command supports async_ arg we can use the above... + self.cmd_p = now_proc(self.cmdrepl.run_command(cmds, timeout=120)) + + # stdout and err both combined into logfile from the spawned repl + stdout = os.path.join(self.rundir, "_cmdcon-log.txt") + self.pytest_hook_run_cmd(stdout, None) + else: + # If we only have a console we can't run in parallel, so run to completion + self.cmd_p = now_proc(self.conrepl.run_command(cmds, timeout=120)) + + return self.cmd_p + + # InterfaceMixin override + # We need a name unique in the shared namespace. + def get_ns_ifname(self, ifname): + return self.name + ifname + + async def add_host_intf(self, hname, lname, mtu=None): + # L3QemuVM needs it's own add_host_intf for macvtap, We need to create the tap + # in the host then move that interface so that the ifindex/devfile are + # different. + + if hname in self.host_intfs: + return + + self.host_intfs[hname] = lname + index = len(self.host_intfs) + + tapindex = self.unet.tapcount + self.unet.tapcount = self.unet.tapcount + 1 + + tapname = f"tap{tapindex}" + self.tapnames[hname] = tapname + + mac = f"02:bb:bb:bb:{index:02x}:{self.id:02x}" + self.tapmacs[hname] = mac + + self.unet.rootcmd.cmd_raises( + f"ip link add link {hname} name {tapname} type macvtap" + ) + if mtu: + self.unet.rootcmd.cmd_raises(f"ip link set {tapname} mtu {mtu}") + self.unet.rootcmd.cmd_raises(f"ip link set {tapname} address {mac} up") + ifindex = self.unet.rootcmd.cmd_raises( + f"cat /sys/class/net/{tapname}/ifindex" + ).strip() + # self.unet.rootcmd.cmd_raises(f"ip link set {tapname} netns {self.pid}") + + tapfile = f"/dev/tap{ifindex}" + fd = os.open(tapfile, os.O_RDWR) + self.tapfds[hname] = fd + self.logger.info( + "%s: Add host intf: created macvtap interface %s (%s) on %s fd %s", + self, + tapname, + tapfile, + hname, + fd, + ) + + async def rem_host_intf(self, hname): + tapname = self.tapnames[hname] + self.unet.rootcmd.cmd_raises(f"ip link set {tapname} down") + self.unet.rootcmd.cmd_raises(f"ip link delete {tapname} type macvtap") + del self.tapnames[hname] + del self.host_intfs[hname] + + async def create_tap(self, index, ifname, mtu=None, driver="virtio-net-pci"): + # XXX we shouldn't be doign a tap on a bridge with a veth + # we should just be using a tap created earlier which was connected to the + # bridge. Except we need to handle the case of p2p qemu <-> namespace + # + ifname = self.get_ns_ifname(ifname) + brname = f"{self.name}br{index}" + + tapindex = self.unet.tapcount + self.unet.tapcount += 1 + + mac = f"02:aa:aa:aa:{index:02x}:{self.id:02x}" + # nic = "tap,model=virtio-net-pci" + # qemu -net nic,model=virtio,addr=1a:46:0b:ca:bc:7b -net tap,fd=3 3<>/dev/tap11 + self.cmd_raises(f"ip address flush dev {ifname}") + self.cmd_raises(f"ip tuntap add tap{tapindex} mode tap") + self.cmd_raises(f"ip link add name {brname} type bridge") + self.cmd_raises(f"ip link set dev {ifname} master {brname}") + self.cmd_raises(f"ip link set dev tap{tapindex} master {brname}") + if mtu: + self.cmd_raises(f"ip link set dev tap{tapindex} mtu {mtu}") + self.cmd_raises(f"ip link set dev {ifname} mtu {mtu}") + self.cmd_raises(f"ip link set dev tap{tapindex} up") + self.cmd_raises(f"ip link set dev {ifname} up") + self.cmd_raises(f"ip link set dev {brname} up") + dev = f"{driver},netdev=n{index},mac={mac}" + return [ + "-netdev", + f"tap,id=n{index},ifname=tap{tapindex},script=no,downscript=no", + "-device", + dev, + ] + + async def mount_mounts(self): + """Mount any shared directories.""" + self.logger.info("Mounting shared directories") + con = self.conrepl + for i, m in enumerate(self.extra_mounts): + outer, mp, uargs = m + if not outer: + con.cmd_raises(f"mkdir -p {mp}") + margs = f"-o {uargs}" if uargs else "" + con.cmd_raises(f"mount {margs} -t tmpfs tmpfs {mp}") + continue + + uargs = "" if uargs is None else uargs + margs = "trans=virtio" + if uargs: + margs += f",{uargs}" + self.logger.info("Mounting %s on %s with %s", outer, mp, margs) + con.cmd_raises(f"mkdir -p {mp}") + con.cmd_raises(f"mount -t 9p -o {margs} shared{i} {mp}") + + async def renumber_interfaces(self): + """Re-number the interfaces. + + After VM comes up need to renumber the interfaces now on the inside. + """ + self.logger.info("Renumbering interfaces") + con = self.conrepl + con.cmd_raises("sysctl -w net.ipv4.ip_forward=1") + if self.unet.ipv6_enable: + self.cmd_raises("sysctl -w net.ipv6.conf.all.forwarding=1") + for ifname in sorted(self.intfs): + conn = find_with_kv(self.config.get("connections"), "name", ifname) + to = conn["to"] + switch = self.unet.switches.get(to) + mtu = conn.get("mtu") + if not mtu and switch: + mtu = switch.config.get("mtu") + if mtu: + con.cmd_raises(f"ip link set {ifname} mtu {mtu}") + con.cmd_raises(f"ip link set {ifname} up") + # In case there was some preconfig e.g., cloud-init + con.cmd_raises(f"ip -4 addr flush dev {ifname}") + sw_is_nat = switch and hasattr(switch, "is_nat") and switch.is_nat + if ifaddr := self.get_intf_addr(ifname, ipv6=False): + con.cmd_raises(f"ip addr add {ifaddr} dev {ifname}") + if sw_is_nat: + # In case there was some preconfig e.g., cloud-init + con.cmd_raises("ip route flush exact default") + con.cmd_raises(f"ip route add default via {switch.ip_address}") + if ifaddr := self.get_intf_addr(ifname, ipv6=True): + con.cmd_raises(f"ip -6 addr add {ifaddr} dev {ifname}") + if sw_is_nat: + # In case there was some preconfig e.g., cloud-init + con.cmd_raises("ip -6 route flush exact default") + con.cmd_raises(f"ip -6 route add default via {switch.ip6_address}") + con.cmd_raises("ip link set lo up") + + if self.unet.cfgopt.getoption("--coverage"): + con.cmd_raises("mount -t debugfs none /sys/kernel/debug") + + async def gather_coverage_data(self): + con = self.conrepl + + gcda = "/sys/kernel/debug/gcov" + tmpdir = con.cmd_raises("mktemp -d").strip() + dest = "/gcov-data.tgz" + con.cmd_raises(rf"find {gcda} -type d -exec mkdir -p {tmpdir}/{{}} \;") + con.cmd_raises( + rf"find {gcda} -name '*.gcda' -exec sh -c 'cat < $0 > {tmpdir}/$0' {{}} \;" + ) + con.cmd_raises( + rf"find {gcda} -name '*.gcno' -exec sh -c 'cp -d $0 {tmpdir}/$0' {{}} \;" + ) + con.cmd_raises(rf"tar cf - -C {tmpdir} sys | gzip -c > {dest}") + con.cmd_raises(rf"rm -rf {tmpdir}") + self.logger.info("Saved coverage data in VM at %s", dest) + if self.use_ssh: + ldest = os.path.join(self.rundir, "gcov-data.tgz") + self.cmd_raises(["/bin/cat", dest], stdout=open(ldest, "wb")) + self.logger.info("Saved coverage data on host at %s", ldest) + + async def _opencons( + self, + *cnames, + prompt=None, + is_bourne=True, + user="root", + password="", + expects=None, + sends=None, + timeout=-1, + ): + """Open consoles based on socket file names.""" + timeo = Timeout(timeout) + cons = [] + for cname in cnames: + sockpath = os.path.join(self.sockdir, cname) + connected = False + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + while self.launch_p.returncode is None and not timeo.is_expired(): + try: + sock.connect(sockpath) + connected = True + break + except OSError as error: + if error.errno == errno.ENOENT: + self.logger.debug("waiting for console socket: %s", sockpath) + else: + self.logger.warning( + "can't open console socket: %s", error.strerror + ) + raise + elapsed = int(timeo.elapsed()) + if elapsed <= 3: + await asyncio.sleep(0.25) + else: + self.logger.info( + "%s: launch (qemu) taking more than %ss", self, elapsed + ) + await asyncio.sleep(1) + + if connected: + if prompt is None: + prompt = r"(^|\r\n)[^#\$]*[#\$] " + cons.append( + await self.console( + sock, + prompt=prompt, + is_bourne=is_bourne, + user=user, + password=password, + use_pty=False, + logfile_prefix=cname, + will_echo=True, + expects=expects, + sends=sends, + timeout=timeout, + trace=True, + ) + ) + elif self.launch_p.returncode is not None: + self.logger.warning( + "%s: launch (qemu) exited quickly (%ss) rc: %s", + self, + timeo.elapsed(), + self.launch_p.returncode, + ) + raise Exception("Qemu launch exited early") + elif timeo.is_expired(): + self.logger.critical( + "%s: timeout (%ss) waiting for qemu to start", + self, + timeo.elapsed(), + ) + assert not timeo.is_expired() + + return cons + + async def set_cpu_affinity(self, afflist): + for i, aff in enumerate(afflist): + if not aff: + continue + # affmask = convert_ranges_to_bitmask(aff) + if i not in self.cpu_thread_map: + logging.warning("affinity %s given for missing vcpu %s", aff, i) + continue + logging.info("setting vcpu %s affinity to %s", i, aff) + tid = self.cpu_thread_map[i] + self.cmd_raises_nsonly(f"taskset -cp {aff} {tid}") + + async def launch(self): + """Launch qemu.""" + self.logger.info("%s: Launch Qemu", self) + + qc = self.qemu_config + cc = qc.get("console", {}) + bootd = "d" if "iso" in qc else "c" + # args = [get_exec_path_host("qemu-system-x86_64"), + # "-nodefaults", "-boot", bootd] + args = [get_exec_path_host("qemu-system-x86_64"), "-boot", bootd] + + args += ["-machine", "q35"] + + if qc.get("kvm"): + rc, _, e = await self.async_cmd_status_nsonly("ls -l /dev/kvm") + if rc: + self.logger.warning("Can't enable KVM no /dev/kvm: %s", e) + else: + # [args += ["-enable-kvm", "-cpu", "host"] + # uargs += ["-accel", "kvm", "-cpu", "Icelake-Server-v5"] + args += ["-accel", "kvm", "-cpu", "host"] + + if ncpu := qc.get("ncpu"): + # args += ["-smp", f"sockets={ncpu}"] + args += ["-smp", f"cores={ncpu}"] + # args += ["-smp", f"{ncpu},sockets={ncpu},cores=1,threads=1"] + + args.extend(["-m", str(qc.get("memory", "512M"))]) + + if "bios" in qc: + if qc["bios"] == "open-firmware": + args.extend(["-bios", "/usr/share/qemu/OVMF.fd"]) + else: + args.extend(["-bios", qc["bios"]]) + if "kernel" in qc: + args.extend(["-kernel", qc["kernel"]]) + if "initrd" in qc: + args.extend(["-initrd", qc["initrd"]]) + if "iso" in qc: + args.extend(["-cdrom", qc["iso"]]) + + # we only have append if we have a kernel + if "kernel" in qc: + args.append("-append") + root = qc.get("root", "/dev/ram0") + # Only 1 serial console the other ports (ttyS[123] hvc[01]) should have + # gettys in inittab + append = f"root={root} rw console=ttyS0" + if "cmdline-extra" in qc: + append += f" {qc['cmdline-extra']}" + args.append(append) + + if "extra-args" in qc: + if isinstance(qc["extra-args"], list): + args.extend(qc["extra-args"]) + else: + args.extend(shlex.split(qc["extra-args"])) + + # Walk the list of connections in order so we attach them the same way + pass_fds = [] + nnics = 0 + pciaddr = 3 + for index, conn in enumerate(self.config["connections"]): + devaddr = conn.get("physical", "") + hostintf = conn.get("hostintf", "") + if devaddr: + # if devaddr in self.tapmacs: + # mac = f",mac={self.tapmacs[devaddr]}" + # else: + # mac = "" + args += ["-device", f"vfio-pci,host={devaddr},addr={pciaddr}"] + elif hostintf: + fd = self.tapfds[hostintf] + mac = self.tapmacs[hostintf] + args += [ + "-nic", + f"tap,model=virtio-net-pci,mac={mac},fd={fd},addr={pciaddr}", + ] + pass_fds.append(fd) + nnics += 1 + elif not hostintf: + driver = conn.get("driver", "virtio-net-pci") + mtu = conn.get("mtu") + if not mtu and conn["to"] in self.unet.switches: + mtu = self.unet.switches[conn["to"]].config.get("mtu") + tapargs = await self.create_tap( + index, conn["name"], mtu=mtu, driver=driver + ) + tapargs[-1] += f",addr={pciaddr}" + args += tapargs + nnics += 1 + pciaddr += 1 + if not nnics: + args += ["-nic", "none"] + + dtpl = qc.get("disk-template") + diskpath = disk = qc.get("disk") + if dtpl and not disk: + disk = qc["disk"] = f"{self.name}-{os.path.basename(dtpl)}" + diskpath = os.path.join(self.rundir, disk) + if self.path_exists(diskpath): + logging.debug("Disk '%s' file exists, using.", diskpath) + else: + dtplpath = os.path.abspath( + os.path.join( + os.path.dirname(self.unet.config["config_pathname"]), dtpl + ) + ) + logging.info("Create disk '%s' from template '%s'", diskpath, dtplpath) + self.cmd_raises( + f"qemu-img create -f qcow2 -F qcow2 -b {dtplpath} {diskpath}" + ) + + if diskpath: + args.extend( + ["-drive", f"file={diskpath},if=none,id=sata-disk0,format=qcow2"] + ) + args.extend(["-device", "ahci,id=ahci"]) + args.extend(["-device", "ide-hd,bus=ahci.0,drive=sata-disk0"]) + + use_stdio = cc.get("stdio", True) + has_cmd = self.config.get("cmd") + use_cmdcon = has_cmd and use_stdio + + # + # Any extra serial/console ports beyond thw first, require entries in + # inittab to have getty running on them, modify inittab + # + # Use -serial stdio for output only, and as the first serial console + # which kernel uses for printk, as it has serious issues with dropped + # input chars for some reason. + # + # 4 serial ports (max), we'll add extra ports using virtual consoles. + _sd = self.sockdir + if use_stdio: + args += ["-serial", "stdio"] + args += ["-serial", f"unix:{_sd}/_console,server,nowait"] + if use_cmdcon: + args += [ + "-serial", + f"unix:{_sd}/_cmdcon,server,nowait", + ] + args += [ + "-serial", + f"unix:{_sd}/console,server,nowait", + # A 2 virtual consoles - /dev/hvc[01] + # Requires CONFIG_HVC_DRIVER=y CONFIG_VIRTIO_CONSOLE=y + "-device", + "virtio-serial", # serial console bus + "-chardev", + f"socket,path={_sd}/vcon0,server=on,wait=off,id=vcon0", + "-chardev", + f"socket,path={_sd}/vcon1,server=on,wait=off,id=vcon1", + "-device", + "virtconsole,chardev=vcon0", + "-device", + "virtconsole,chardev=vcon1", + # 2 monitors + "-monitor", + f"unix:{_sd}/_monitor,server,nowait", + "-monitor", + f"unix:{_sd}/monitor,server,nowait", + "-gdb", + f"unix:{_sd}/gdbserver,server,nowait", + ] + + for i, m in enumerate(self.extra_mounts): + args += [ + "-virtfs", + f"local,path={m[0]},mount_tag=shared{i},security_model=passthrough", + ] + + args += ["-nographic"] + + # + # Launch Qemu + # + + stdout = open(os.path.join(self.rundir, "qemu.out"), "wb") + stderr = open(os.path.join(self.rundir, "qemu.err"), "wb") + self.launch_p = await self.async_popen( + args, + stdin=subprocess.DEVNULL, + stdout=stdout, + stderr=stderr, + pass_fds=pass_fds, + # We don't need this here b/c we are only ever running qemu and that's all + # we need to kill for cleanup + # XXX reconcile this + start_new_session=True, # allows us to signal all children to exit + ) + + self.pytest_hook_run_cmd(stdout, stderr) + + # We've passed these on, so don't need these open here anymore. + for fd in pass_fds: + os.close(fd) + + self.logger.debug("%s: async_popen => %s", self, self.launch_p.pid) + + confiles = ["_console"] + if use_cmdcon: + confiles.append("_cmdcon") + + # + # Connect to the console socket, retrying + # + prompt = cc.get("prompt") + cons = await self._opencons( + *confiles, + prompt=prompt, + is_bourne=not bool(prompt), + user=cc.get("user", "root"), + password=cc.get("password", ""), + expects=cc.get("expects"), + sends=cc.get("sends"), + timeout=int(cc.get("timeout", 60)), + ) + self.conrepl = cons[0] + if use_cmdcon: + self.cmdrepl = cons[1] + self.monrepl = await self.monitor(os.path.join(self.sockdir, "_monitor")) + + # the monitor output has super annoying ANSI escapes in it + + output = self.monrepl.cmd_nostatus("info status") + self.logger.info("VM status: %s", output) + + output = self.monrepl.cmd_nostatus("info kvm") + self.logger.info("KVM status: %s", output) + + # + # Set thread affinity + # + output = self.monrepl.cmd_nostatus("info cpus") + matches = re.findall(r"CPU #(\d+): *thread_id=(\d+)", output) + self.cpu_thread_map = {int(k): int(v) for k, v in matches} + if cpuaff := self.qemu_config.get("cpu-affinity"): + await self.set_cpu_affinity(cpuaff) + + self.is_kvm = "disabled" not in output + + if qc.get("unix-os", True): + await self.renumber_interfaces() + + if self.extra_mounts: + await self.mount_mounts() + + self.use_ssh = bool(self.ssh_keyfile) + if self.use_ssh: + self.use_ssh = self.__setup_ssh() + + self.pytest_hook_open_shell() + + return self.launch_p + + def launch_completed(self, future): + self.logger.debug("%s: launch (qemu) completed called", self) + self.use_ssh = False + try: + n = future.result() + self.logger.debug("%s: node launch (qemu) completed result: %s", self, n) + except asyncio.CancelledError as error: + self.logger.debug( + "%s: node launch (qemu) cmd wait() canceled: %s", future, error + ) + + async def cleanup_qemu(self): + """Launch qemu.""" + if self.launch_p: + await self.async_cleanup_proc(self.launch_p) + + async def async_cleanup_cmd(self): + """Run the configured cleanup commands for this node.""" + self.cleanup_called = True + + if "cleanup-cmd" not in self.config: + return + + if not self.launch_p: + self.logger.warning("async_cleanup_cmd: qemu no longer running") + return + + raise NotImplementedError("Needs to be like run_cmd") + # return await self._async_cleanup_cmd() + + async def _async_delete(self): + self.logger.debug("%s: deleting", self) + + # Need to cleanup early b/c it is running on the VM + if self.cmd_p: + await self.async_cleanup_proc(self.cmd_p) + self.cmd_p = None + + try: + # Need to cleanup early b/c it is running on the VM + if not self.cleanup_called: + await self.async_cleanup_cmd() + except Exception as error: + self.logger.warning( + "Got an error during delete from async_cleanup_cmd: %s", error + ) + + try: + if not self.launch_p: + self.logger.warning("async_delete: qemu is not running") + else: + await self.cleanup_qemu() + except Exception as error: + self.logger.warning("%s: failued to cleanup qemu process: %s", self, error) + + await super()._async_delete() + + +class Munet(BaseMunet): + """Munet.""" + + def __init__( + self, + rundir=None, + config=None, + pid=True, + logger=None, + **kwargs, + ): + # logging.warning("Munet") + + if not rundir: + rundir = "/tmp/munet" + + if logger is None: + logger = logging.getLogger("munet.unet") + + super().__init__("munet", pid=pid, rundir=rundir, logger=logger, **kwargs) + + self.built = False + self.tapcount = 0 + + self.cmd_raises(f"mkdir -p {self.rundir} && chmod 755 {self.rundir}") + self.set_ns_cwd(self.rundir) + + if not config: + config = {} + self.config = config + if "config_pathname" in config: + self.config_pathname = os.path.realpath(config["config_pathname"]) + self.config_dirname = os.path.dirname(self.config_pathname) + else: + self.config_pathname = "" + self.config_dirname = "" + + # Done in BaseMunet now + # # We need some way to actually get back to the root namespace + # if not self.isolated: + # self.rootcmd = commander + # else: + # spid = str(pid) + # nsflags = (f"--mount={self.proc_path / spid / 'ns/mnt'}", + # f"--net={self.proc_path / spid / 'ns/net'}", + # f"--uts={self.proc_path / spid / 'ns/uts'}", + # f"--ipc={self.proc_path / spid / 'ns/ipc'}", + # f"--cgroup={self.proc_path / spid / 'ns/cgroup'}", + # f"--pid={self.proc_path / spid / 'ns/net'}", + # self.rootcmd = SharedNamespace("host", pid=1, nsflags=nsflags) + + # Save the namespace pid + with open(os.path.join(self.rundir, "nspid"), "w", encoding="ascii") as f: + f.write(f"{self.pid}\n") + + with open(os.path.join(self.rundir, "nspids"), "w", encoding="ascii") as f: + f.write(f'{" ".join([str(x) for x in self.pids])}\n') + + hosts_file = os.path.join(self.rundir, "hosts.txt") + with open(hosts_file, "w", encoding="ascii") as hf: + hf.write( + f"""127.0.0.1\tlocalhost {self.name} +::1\tip6-localhost ip6-loopback +fe00::0\tip6-localnet +ff00::0\tip6-mcastprefix +ff02::1\tip6-allnodes +ff02::2\tip6-allrouters +""" + ) + self.bind_mount(hosts_file, "/etc/hosts") + + # Common CLI commands for any topology + cdict = { + "commands": [ + { + "name": "pcap", + "format": "pcap NETWORK", + "help": ( + "capture packets from NETWORK into file capture-NETWORK.pcap" + " the command is run within a new window which also shows" + " packet summaries. NETWORK can also be an interface specified" + " as HOST:INTF. To capture inside the host namespace." + ), + "exec": "tshark -s 9200 -i {0} -P -w capture-{0}.pcap", + "top-level": True, + "new-window": {"background": True}, + }, + { + "name": "nsterm", + "format": "nsterm HOST [HOST ...]", + "help": ( + "open terminal[s] in the namespace only" + " (outside containers or VM), * for all" + ), + "exec": "bash", + "new-window": {"ns_only": True}, + }, + { + "name": "term", + "format": "term HOST [HOST ...]", + "help": "open terminal[s] (TMUX or XTerm) on HOST[S], * for all", + "exec": "bash", + "new-window": True, + }, + { + "name": "xterm", + "format": "xterm HOST [HOST ...]", + "help": "open XTerm[s] on HOST[S], * for all", + "exec": "bash", + "new-window": { + "forcex": True, + }, + }, + { + "name": "sh", + "format": "[HOST ...] sh <SHELL-COMMAND>", + "help": "execute <SHELL-COMMAND> on hosts", + "exec": "{}", + }, + { + "name": "shi", + "format": "[HOST ...] shi <INTERACTIVE-COMMAND>", + "help": "execute <INTERACTIVE-COMMAND> on HOST[s]", + "exec": "{}", + "interactive": True, + }, + { + "name": "stdout", + "exec": ( + "[ -e %RUNDIR%/qemu.out ] && tail -F %RUNDIR%/qemu.out " + "|| tail -F %RUNDIR%/cmd.out" + ), + "format": "stdout HOST [HOST ...]", + "help": "tail -f on the stdout of the qemu/cmd for this node", + "new-window": True, + }, + { + "name": "stderr", + "exec": ( + "[ -e %RUNDIR%/qemu.err ] && tail -F %RUNDIR%/qemu.err " + "|| tail -F %RUNDIR%/cmd.err" + ), + "format": "stderr HOST [HOST ...]", + "help": "tail -f on the stdout of the qemu/cmd for this node", + "new-window": True, + }, + ] + } + + cli.add_cli_config(self, cdict) + + if "cli" in config: + cli.add_cli_config(self, config["cli"]) + + if "topology" not in self.config: + self.config["topology"] = {} + + self.topoconf = self.config["topology"] + self.ipv6_enable = self.topoconf.get("ipv6-enable", False) + + if self.isolated: + if not self.ipv6_enable: + # Disable IPv6 + self.cmd_raises("sysctl -w net.ipv6.conf.all.autoconf=0") + self.cmd_raises("sysctl -w net.ipv6.conf.all.disable_ipv6=1") + else: + self.cmd_raises("sysctl -w net.ipv6.conf.all.autoconf=1") + self.cmd_raises("sysctl -w net.ipv6.conf.all.disable_ipv6=0") + + # we really need overlay, but overlay-layers (used by overlay-images) + # counts on things being present in overlay so this temp stuff doesn't work. + # if self.isolated: + # # Let's hide podman details + # self.tmpfs_mount("/var/lib/containers/storage/overlay-containers") + + shellopt = self.cfgopt.getoption("--shell") + shellopt = shellopt if shellopt else "" + if shellopt == "all" or "." in shellopt.split(","): + self.run_in_window("bash") + + def __del__(self): + """Catch case of build object but not async_deleted.""" + if hasattr(self, "built"): + if not self.deleting: + logging.critical( + "Munet object deleted without calling `async_delete` for cleanup." + ) + s = super() + if hasattr(s, "__del__"): + s.__del__(self) + + async def _async_build(self, logger=None): + """Build the topology based on config.""" + if self.built: + self.logger.warning("%s: is already built", self) + return + + self.built = True + + # Allow for all networks to be auto-numbered + topoconf = self.topoconf + autonumber = self.autonumber + ipv6_enable = self.ipv6_enable + + # --------------------------------------------- + # Merge Kinds and perform variable substitution + # --------------------------------------------- + + kinds = self.config.get("kinds", {}) + + for name, conf in config_to_dict_with_key(topoconf, "networks", "name").items(): + if kind := conf.get("kind"): + if kconf := kinds[kind]: + conf = merge_kind_config(kconf, conf) + conf = config_subst( + conf, name=name, rundir=self.rundir, configdir=self.config_dirname + ) + if "ip" not in conf and autonumber: + conf["ip"] = "auto" + if "ipv6" not in conf and autonumber and ipv6_enable: + conf["ipv6"] = "auto" + topoconf["networks"][name] = conf + self.add_network(name, conf, logger=logger) + + for name, conf in config_to_dict_with_key(topoconf, "nodes", "name").items(): + if kind := conf.get("kind"): + if kconf := kinds[kind]: + conf = merge_kind_config(kconf, conf) + + config_to_dict_with_key( + conf, "env", "name" + ) # convert list of env objects to dict + + conf = config_subst( + conf, + name=name, + rundir=os.path.join(self.rundir, name), + configdir=self.config_dirname, + ) + topoconf["nodes"][name] = conf + self.add_l3_node(name, conf, logger=logger) + + # ------------------ + # Create connections + # ------------------ + + # Go through all connections and name them so they are sane to the user + # otherwise when we do p2p links the names/ords skip around based oddly + for name, node in self.hosts.items(): + nconf = node.config + if "connections" not in nconf: + continue + nconns = [] + for cconf in nconf["connections"]: + # Replace string only with a dictionary + if isinstance(cconf, str): + splitconf = cconf.split(":", 1) + cconf = {"to": splitconf[0]} + if len(splitconf) == 2: + cconf["name"] = splitconf[1] + # Allocate a name if not already assigned + if "name" not in cconf: + cconf["name"] = node.get_next_intf_name() + nconns.append(cconf) + nconf["connections"] = nconns + + for name, node in self.hosts.items(): + nconf = node.config + if "connections" not in nconf: + continue + for cconf in nconf["connections"]: + # Eventually can add support for unconnected intf here. + if "to" not in cconf: + continue + to = cconf["to"] + if to in self.switches: + switch = self.switches[to] + swconf = find_matching_net_config(name, cconf, switch.config) + await self.add_native_link(switch, node, swconf, cconf) + elif cconf["name"] not in node.intfs: + # Only add the p2p interface if not already there. + other = self.hosts[to] + oconf = find_matching_net_config(name, cconf, other.config) + await self.add_native_link(node, other, cconf, oconf) + + @property + def autonumber(self): + return self.topoconf.get("networks-autonumber", False) + + @autonumber.setter + def autonumber(self, value): + self.topoconf["networks-autonumber"] = bool(value) + + async def add_native_link(self, node1, node2, c1=None, c2=None): + """Add a link between switch and node or 2 nodes.""" + isp2p = False + + c1 = {} if c1 is None else c1 + c2 = {} if c2 is None else c2 + + if node1.name in self.switches: + assert node2.name in self.hosts + elif node2.name in self.switches: + assert node1.name in self.hosts + node1, node2 = node2, node1 + c1, c2 = c2, c1 + else: + # p2p link + assert node1.name in self.hosts + assert node1.name in self.hosts + isp2p = True + + if "name" not in c1: + c1["name"] = node1.get_next_intf_name() + if1 = c1["name"] + + if "name" not in c2: + c2["name"] = node2.get_next_intf_name() + if2 = c2["name"] + + do_add_link = True + for n, c in ((node1, c1), (node2, c2)): + if "hostintf" in c: + await n.add_host_intf(c["hostintf"], c["name"], mtu=c.get("mtu")) + do_add_link = False + elif "physical" in c: + await n.add_phy_intf(c["physical"], c["name"]) + do_add_link = False + if do_add_link: + assert "hostintf" not in c1 + assert "hostintf" not in c2 + assert "physical" not in c1 + assert "physical" not in c2 + + if isp2p: + mtu1 = c1.get("mtu") + mtu2 = c2.get("mtu") + mtu = mtu1 if mtu1 else mtu2 + if mtu1 and mtu2 and mtu1 != mtu2: + self.logger.error("mtus differ for add_link %s != %s", mtu1, mtu2) + else: + mtu = c2.get("mtu") + + super().add_link(node1, node2, if1, if2, mtu=mtu) + + if isp2p: + node1.set_p2p_addr(node2, c1, c2) + else: + node2.set_lan_addr(node1, c2) + + if "physical" not in c1 and not node1.is_vm: + node1.set_intf_constraints(if1, **c1) + if "physical" not in c2 and not node2.is_vm: + node2.set_intf_constraints(if2, **c2) + + def add_l3_node(self, name, config=None, **kwargs): + """Add a node to munet.""" + if config and config.get("image"): + cls = L3ContainerNode + elif config and config.get("qemu"): + cls = L3QemuVM + elif config and config.get("server"): + cls = SSHRemote + kwargs["server"] = config["server"] + kwargs["port"] = int(config.get("server-port", 22)) + if "ssh-identity-file" in config: + kwargs["idfile"] = config.get("ssh-identity-file") + if "ssh-user" in config: + kwargs["user"] = config.get("ssh-user") + if "ssh-password" in config: + kwargs["password"] = config.get("ssh-password") + else: + cls = L3NamespaceNode + return super().add_host(name, cls=cls, config=config, **kwargs) + + def add_network(self, name, config=None, **kwargs): + """Add a l2 or l3 switch to munet.""" + if config is None: + config = {} + + cls = L3Bridge if config.get("ip") else L2Bridge + mtu = kwargs.get("mtu", config.get("mtu")) + return super().add_switch(name, cls=cls, config=config, mtu=mtu, **kwargs) + + async def run(self): + tasks = [] + + hosts = self.hosts.values() + launch_nodes = [x for x in hosts if hasattr(x, "launch")] + launch_nodes = [x for x in launch_nodes if x.config.get("qemu")] + run_nodes = [x for x in hosts if hasattr(x, "has_run_cmd") and x.has_run_cmd()] + ready_nodes = [ + x for x in hosts if hasattr(x, "has_ready_cmd") and x.has_ready_cmd() + ] + + pcapopt = self.cfgopt.getoption("--pcap") + pcapopt = pcapopt if pcapopt else "" + if pcapopt == "all": + pcapopt = self.switches.keys() + if pcapopt: + for pcap in pcapopt.split(","): + if ":" in pcap: + host, intf = pcap.split(":") + pcap = f"{host}-{intf}" + host = self.hosts[host] + else: + host = self + intf = pcap + host.run_in_window( + f"tshark -s 9200 -i {intf} -P -w capture-{pcap}.pcap", + background=True, + title=f"cap:{pcap}", + ) + + if launch_nodes: + # would like a info when verbose here. + logging.debug("Launching nodes") + await asyncio.gather(*[x.launch() for x in launch_nodes]) + + # Watch for launched processes to exit + for node in launch_nodes: + task = asyncio.create_task( + node.launch_p.wait(), name=f"Node-{node.name}-launch" + ) + task.add_done_callback(node.launch_completed) + tasks.append(task) + + if run_nodes: + # would like a info when verbose here. + logging.debug("Running `cmd` on nodes") + await asyncio.gather(*[x.run_cmd() for x in run_nodes]) + + # Watch for run_cmd processes to exit + for node in run_nodes: + task = asyncio.create_task(node.cmd_p.wait(), name=f"Node-{node.name}-cmd") + task.add_done_callback(node.cmd_completed) + tasks.append(task) + + # Wait for nodes to be ready + if ready_nodes: + + async def wait_until_ready(x): + while not await x.async_ready_cmd(): + logging.debug("Waiting for ready on: %s", x) + await asyncio.sleep(0.25) + logging.debug("%s is ready!", x) + + logging.debug("Waiting for ready on nodes: %s", ready_nodes) + _, pending = await asyncio.wait( + [wait_until_ready(x) for x in ready_nodes], timeout=30 + ) + if pending: + logging.warning("Timeout waiting for ready: %s", pending) + for nr in pending: + nr.cancel() + raise asyncio.TimeoutError() + logging.debug("All nodes ready") + + return tasks + + async def _async_delete(self): + from .testing.util import async_pause_test # pylint: disable=C0415 + + self.logger.debug("%s: deleting.", self) + + if self.cfgopt.getoption("--coverage"): + nodes = ( + x for x in self.hosts.values() if hasattr(x, "gather_coverage_data") + ) + try: + await asyncio.gather(*(x.gather_coverage_data() for x in nodes)) + except Exception as error: + logging.warning("Error gathering coverage data: %s", error) + + pause = bool(self.cfgopt.getoption("--pause-at-end")) + pause = pause or bool(self.cfgopt.getoption("--pause")) + if pause: + try: + await async_pause_test("Before MUNET delete") + except KeyboardInterrupt: + print("^C...continuing") + except Exception as error: + self.logger.error("\n...continuing after error: %s", error) + + # XXX should we cancel launch and run tasks? + + try: + await super()._async_delete() + except Exception as error: + self.logger.error("Error cleaning up: %s", error, exc_info=True) + raise + + +async def run_cmd_update_ceos(node, shell_cmd, cmds, cmd): + cmd = cmd.strip() + if shell_cmd or cmd != "/sbin/init": + return cmds, cmd + + # + # Add flash dir and mount it + # + flashdir = os.path.join(node.rundir, "flash") + node.cmd_raises_nsonly(f"mkdir -p {flashdir} && chmod 775 {flashdir}") + cmds += [f"--volume={flashdir}:/mnt/flash"] + + # + # Startup config (if not present already) + # + if startup_config := node.config.get("startup-config", None): + dest = os.path.join(flashdir, "startup-config") + if os.path.exists(dest): + node.logger.info("Skipping copy of startup-config, already present") + else: + source = os.path.join(node.unet.config_dirname, startup_config) + node.cmd_raises_nsonly(f"cp {source} {dest} && chmod 664 {dest}") + + # + # system mac address (if not present already + # + dest = os.path.join(flashdir, "system_mac_address") + if os.path.exists(dest): + node.logger.info("Skipping system-mac generation, already present") + else: + random_arista_mac = "00:1c:73:%02x:%02x:%02x" % ( + random.randint(0, 255), + random.randint(0, 255), + random.randint(0, 255), + ) + system_mac = node.config.get("system-mac", random_arista_mac) + with open(dest, "w", encoding="ascii") as f: + f.write(system_mac + "\n") + node.cmd_raises_nsonly(f"chmod 664 {dest}") + + args = [] + + # Pass special args for the environment variables + if "env" in node.config: + args += [f"systemd.setenv={k}={v}" for k, v in node.config["env"].items()] + + return cmds, [cmd] + args + + +# XXX this is only used by the container code +kind_run_cmd_update = {"ceos": run_cmd_update_ceos} diff --git a/tests/topotests/munet/parser.py b/tests/topotests/munet/parser.py new file mode 100644 index 0000000..4fc0c75 --- /dev/null +++ b/tests/topotests/munet/parser.py @@ -0,0 +1,374 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# September 30 2021, Christian Hopps <chopps@labn.net> +# +# Copyright 2021, LabN Consulting, L.L.C. +# +"""A module that implements the standalone parser.""" +import asyncio +import importlib.resources +import json +import logging +import logging.config +import os +import subprocess +import sys +import tempfile + +from pathlib import Path + + +try: + import jsonschema # pylint: disable=C0415 + import jsonschema.validators # pylint: disable=C0415 + + from jsonschema.exceptions import ValidationError # pylint: disable=C0415 +except ImportError: + jsonschema = None + +from .config import list_to_dict_with_key +from .native import Munet + + +def get_schema(): + if get_schema.schema is None: + with importlib.resources.path("munet", "munet-schema.json") as datapath: + search = [str(datapath.parent)] + get_schema.schema = get_config(basename="munet-schema", search=search) + return get_schema.schema + + +get_schema.schema = None + +project_root_contains = [ + ".git", + "pyproject.toml", + "tox.ini", + "setup.cfg", + "setup.py", + "pytest.ini", + ".projectile", +] + + +def is_project_root(path: Path) -> bool: + + for contains in project_root_contains: + if path.joinpath(contains).exists(): + return True + return False + + +def find_project_root(config_path: Path, project_root=None): + if project_root is not None: + project_root = Path(project_root) + if project_root in config_path.parents: + return project_root + logging.warning( + "project_root %s is not a common ancestor of config file %s", + project_root, + config_path, + ) + return config_path.parent + for ppath in config_path.parents: + if is_project_root(ppath): + return ppath + return config_path.parent + + +def get_config(pathname=None, basename="munet", search=None, logf=logging.debug): + + cwd = os.getcwd() + + if not search: + search = [cwd] + elif isinstance(search, (str, Path)): + search = [search] + + if pathname: + pathname = os.path.join(cwd, pathname) + if not os.path.exists(pathname): + raise FileNotFoundError(pathname) + else: + for d in search: + logf("%s", f'searching in "{d}" for "{basename}".{{yaml, toml, json}}') + for ext in ("yaml", "toml", "json"): + pathname = os.path.join(d, basename + "." + ext) + if os.path.exists(pathname): + logf("%s", f'Found "{pathname}"') + break + else: + continue + break + else: + raise FileNotFoundError(basename + ".{json,toml,yaml} in " + f"{search}") + + _, ext = pathname.rsplit(".", 1) + + if ext == "json": + config = json.load(open(pathname, encoding="utf-8")) + elif ext == "toml": + import toml # pylint: disable=C0415 + + config = toml.load(pathname) + elif ext == "yaml": + import yaml # pylint: disable=C0415 + + config = yaml.safe_load(open(pathname, encoding="utf-8")) + else: + raise ValueError("Filename does not end with (.json|.toml|.yaml)") + + config["config_pathname"] = os.path.realpath(pathname) + return config + + +def setup_logging(args, config_base="logconf"): + # Create rundir and arrange for future commands to run in it. + + # Change CWD to the rundir prior to parsing config + old = os.getcwd() + os.chdir(args.rundir) + try: + search = [old] + with importlib.resources.path("munet", config_base + ".yaml") as datapath: + search.append(str(datapath.parent)) + + def logf(msg, *p, **k): + if args.verbose: + print("PRELOG: " + msg % p, **k, file=sys.stderr) + + config = get_config(args.log_config, config_base, search, logf=logf) + pathname = config["config_pathname"] + del config["config_pathname"] + + if "info_console" in config["handlers"]: + # mutest case + if args.verbose > 1: + config["handlers"]["console"]["level"] = "DEBUG" + config["handlers"]["info_console"]["level"] = "DEBUG" + elif args.verbose: + config["handlers"]["console"]["level"] = "INFO" + config["handlers"]["info_console"]["level"] = "DEBUG" + elif args.verbose: + # munet case + config["handlers"]["console"]["level"] = "DEBUG" + + # add the rundir path to the filenames + for v in config["handlers"].values(): + filename = v.get("filename") + if not filename: + continue + v["filename"] = os.path.join(args.rundir, filename) + + logging.config.dictConfig(dict(config)) + logging.info("Loaded logging config %s", pathname) + + return config + finally: + os.chdir(old) + + +def append_hosts_files(unet, netname): + if not netname: + return + + entries = [] + for name in ("munet", *list(unet.hosts)): + if name == "munet": + node = unet.switches[netname] + ifname = None + else: + node = unet.hosts[name] + if not hasattr(node, "_intf_addrs"): + continue + ifname = node.get_ifname(netname) + + for b in (False, True): + ifaddr = node.get_intf_addr(ifname, ipv6=b) + if ifaddr and hasattr(ifaddr, "ip"): + entries.append((name, ifaddr.ip)) + + for name in ("munet", *list(unet.hosts)): + node = unet if name == "munet" else unet.hosts[name] + if not hasattr(node, "rundir"): + continue + with open(os.path.join(node.rundir, "hosts.txt"), "a+", encoding="ascii") as hf: + hf.write("\n") + for e in entries: + hf.write(f"{e[1]}\t{e[0]}\n") + + +def validate_config(config, logger, args): + if jsonschema is None: + logger.debug("No validation w/o jsonschema module") + return True + + old = os.getcwd() + if args: + os.chdir(args.rundir) + + try: + validator = jsonschema.validators.Draft202012Validator(get_schema()) + validator.validate(instance=config) + logger.debug("Validated %s", config["config_pathname"]) + return True + except FileNotFoundError as error: + logger.info("No schema found: %s", error) + return False + except ValidationError as error: + logger.info("Validation failed: %s", error) + return False + finally: + if args: + os.chdir(old) + + +def load_kinds(args, search=None): + # Change CWD to the rundir prior to parsing config + cwd = os.getcwd() + if args: + os.chdir(args.rundir) + + args_config = args.kinds_config if args else None + try: + if search is None: + search = [cwd] + with importlib.resources.path("munet", "kinds.yaml") as datapath: + search.insert(0, str(datapath.parent)) + + configs = [] + if args_config: + configs.append(get_config(args_config, "kinds", search=[])) + else: + # prefer directories at the front of the list + for kdir in search: + try: + configs.append(get_config(basename="kinds", search=[kdir])) + except FileNotFoundError: + continue + + kinds = {} + for config in configs: + # XXX need to fix the issue with `connections: ["net0"]` not validating + # if jsonschema is not None: + # validator = jsonschema.validators.Draft202012Validator(get_schema()) + # validator.validate(instance=config) + + kinds_list = config.get("kinds", []) + kinds_dict = list_to_dict_with_key(kinds_list, "name") + if kinds_dict: + logging.info("Loading kinds config from %s", config["config_pathname"]) + if "kinds" in kinds: + kinds["kinds"].update(**kinds_dict) + else: + kinds["kinds"] = kinds_dict + + cli_list = config.get("cli", {}).get("commands", []) + if cli_list: + logging.info("Loading cli comands from %s", config["config_pathname"]) + if "cli" not in kinds: + kinds["cli"] = {} + if "commands" not in kinds["cli"]: + kinds["cli"]["commands"] = [] + kinds["cli"]["commands"].extend(cli_list) + + return kinds + except FileNotFoundError as error: + # if we have kinds in args but the file doesn't exist, raise the error + if args_config is not None: + raise error + return {} + finally: + if args: + os.chdir(cwd) + + +async def async_build_topology( + config=None, + logger=None, + rundir=None, + args=None, + unshare_inline=False, + pytestconfig=None, + search_root=None, + top_level_pidns=True, +): + + if not rundir: + rundir = tempfile.mkdtemp(prefix="unet") + subprocess.run(f"mkdir -p {rundir} && chmod 755 {rundir}", check=True, shell=True) + + isolated = not args.host if args else True + if not config: + config = get_config(basename="munet") + + # create search directories from common root if given + cpath = Path(config["config_pathname"]).absolute() + project_root = args.project_root if args else None + if not search_root: + search_root = find_project_root(cpath, project_root) + if not search_root: + search = [cpath.parent] + else: + search_root = Path(search_root).absolute() + if search_root in cpath.parents: + search = list(cpath.parents) + if remcount := len(search_root.parents): + search = search[0:-remcount] + + # load kinds along search path and merge into config + kinds = load_kinds(args, search=search) + config_kinds_dict = list_to_dict_with_key(config.get("kinds", []), "name") + config["kinds"] = {**kinds.get("kinds", {}), **config_kinds_dict} + + # mere CLI command from kinds into config as well. + kinds_cli_list = kinds.get("cli", {}).get("commands", []) + config_cli_list = config.get("cli", {}).get("commands", []) + if config_cli_list: + if kinds_cli_list: + config_cli_list.extend(list(kinds_cli_list)) + elif kinds_cli_list: + if "cli" not in config: + config["cli"] = {} + if "commands" not in config["cli"]: + config["cli"]["commands"] = [] + config["cli"]["commands"].extend(list(kinds_cli_list)) + + unet = Munet( + rundir=rundir, + config=config, + pytestconfig=pytestconfig, + isolated=isolated, + pid=top_level_pidns, + unshare_inline=args.unshare_inline if args else unshare_inline, + logger=logger, + ) + + try: + await unet._async_build(logger) # pylint: disable=W0212 + except Exception as error: + logging.critical("Failure building munet topology: %s", error, exc_info=True) + await unet.async_delete() + raise + except KeyboardInterrupt: + await unet.async_delete() + raise + + topoconf = config.get("topology") + if not topoconf: + return unet + + dns_network = topoconf.get("dns-network") + if dns_network: + append_hosts_files(unet, dns_network) + + # Write our current config to the run directory + with open(f"{unet.rundir}/config.json", "w", encoding="utf-8") as f: + json.dump(unet.config, f, indent=2) + + return unet + + +def build_topology(config=None, logger=None, rundir=None, args=None, pytestconfig=None): + return asyncio.run(async_build_topology(config, logger, rundir, args, pytestconfig)) diff --git a/tests/topotests/munet/testing/__init__.py b/tests/topotests/munet/testing/__init__.py new file mode 100644 index 0000000..63cbfab --- /dev/null +++ b/tests/topotests/munet/testing/__init__.py @@ -0,0 +1 @@ +"""Sub-package supporting munet use in pytest.""" diff --git a/tests/topotests/munet/testing/fixtures.py b/tests/topotests/munet/testing/fixtures.py new file mode 100644 index 0000000..25039df --- /dev/null +++ b/tests/topotests/munet/testing/fixtures.py @@ -0,0 +1,447 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# April 22 2022, Christian Hopps <chopps@gmail.com> +# +# Copyright (c) 2022, LabN Consulting, L.L.C +# +"""A module that implements pytest fixtures. + +To use in your project, in your conftest.py add: + + from munet.testing.fixtures import * +""" +import contextlib +import logging +import os + +from pathlib import Path +from typing import Union + +import pytest +import pytest_asyncio + +from ..base import BaseMunet +from ..base import Bridge +from ..base import get_event_loop +from ..cleanup import cleanup_current +from ..cleanup import cleanup_previous +from ..native import L3NodeMixin +from ..parser import async_build_topology +from ..parser import get_config +from .util import async_pause_test +from .util import pause_test + + +@contextlib.asynccontextmanager +async def achdir(ndir: Union[str, Path], desc=""): + odir = os.getcwd() + os.chdir(ndir) + if desc: + logging.debug("%s: chdir from %s to %s", desc, odir, ndir) + try: + yield + finally: + if desc: + logging.debug("%s: chdir back from %s to %s", desc, ndir, odir) + os.chdir(odir) + + +@contextlib.contextmanager +def chdir(ndir: Union[str, Path], desc=""): + odir = os.getcwd() + os.chdir(ndir) + if desc: + logging.debug("%s: chdir from %s to %s", desc, odir, ndir) + try: + yield + finally: + if desc: + logging.debug("%s: chdir back from %s to %s", desc, ndir, odir) + os.chdir(odir) + + +def get_test_logdir(nodeid=None, module=False): + """Get log directory relative pathname.""" + xdist_worker = os.getenv("PYTEST_XDIST_WORKER", "") + mode = os.getenv("PYTEST_XDIST_MODE", "no") + + # nodeid: all_protocol_startup/test_all_protocol_startup.py::test_router_running + # may be missing "::testname" if module is True + if not nodeid: + nodeid = os.environ["PYTEST_CURRENT_TEST"].split(" ")[0] + + cur_test = nodeid.replace("[", "_").replace("]", "_") + if module: + idx = cur_test.rfind("::") + path = cur_test if idx == -1 else cur_test[:idx] + testname = "" + else: + path, testname = cur_test.split("::") + testname = testname.replace("/", ".") + path = path[:-3].replace("/", ".") + + # We use different logdir paths based on how xdist is running. + if mode == "each": + if module: + return os.path.join(path, "worker-logs", xdist_worker) + return os.path.join(path, testname, xdist_worker) + assert mode in ("no", "load", "loadfile", "loadscope"), f"Unknown dist mode {mode}" + return path if module else os.path.join(path, testname) + + +def _push_log_handler(desc, logpath): + logpath = os.path.abspath(logpath) + logging.debug("conftest: adding %s logging at %s", desc, logpath) + root_logger = logging.getLogger() + handler = logging.FileHandler(logpath, mode="w") + fmt = logging.Formatter("%(asctime)s %(levelname)5s: %(message)s") + handler.setFormatter(fmt) + root_logger.addHandler(handler) + return handler + + +def _pop_log_handler(handler): + root_logger = logging.getLogger() + logging.debug("conftest: removing logging handler %s", handler) + root_logger.removeHandler(handler) + + +@contextlib.contextmanager +def log_handler(desc, logpath): + handler = _push_log_handler(desc, logpath) + try: + yield + finally: + _pop_log_handler(handler) + + +# ================= +# Sessions Fixtures +# ================= + + +@pytest.fixture(autouse=True, scope="session") +def session_autouse(): + if "PYTEST_TOPOTEST_WORKER" not in os.environ: + is_worker = False + elif not os.environ["PYTEST_TOPOTEST_WORKER"]: + is_worker = False + else: + is_worker = True + + if not is_worker: + # This is unfriendly to multi-instance + cleanup_previous() + + # We never pop as we want to keep logging + _push_log_handler("session", "/tmp/unet-test/pytest-session.log") + + yield + + if not is_worker: + cleanup_current() + + +# =============== +# Module Fixtures +# =============== + + +@pytest.fixture(autouse=True, scope="module") +def module_autouse(request): + logpath = get_test_logdir(request.node.name, True) + logpath = os.path.join("/tmp/unet-test", logpath, "pytest-exec.log") + with log_handler("module", logpath): + sdir = os.path.dirname(os.path.realpath(request.fspath)) + with chdir(sdir, "module autouse fixture"): + yield + + if BaseMunet.g_unet: + raise Exception("Base Munet was not cleaned up/deleted") + + +@pytest.fixture(scope="module") +def event_loop(): + """Create an instance of the default event loop for the session.""" + loop = get_event_loop() + try: + logging.info("event_loop_fixture: yielding with new event loop watcher") + yield loop + finally: + loop.close() + + +@pytest.fixture(scope="module") +def rundir_module(): + d = os.path.join("/tmp/unet-test", get_test_logdir(module=True)) + logging.debug("conftest: test module rundir %s", d) + return d + + +async def _unet_impl( + _rundir, _pytestconfig, unshare=None, top_level_pidns=None, param=None +): + try: + # Default is not to unshare inline if not specified otherwise + unshare_default = False + pidns_default = True + if isinstance(param, (tuple, list)): + pidns_default = bool(param[2]) if len(param) > 2 else True + unshare_default = bool(param[1]) if len(param) > 1 else False + param = str(param[0]) + elif isinstance(param, bool): + unshare_default = param + param = None + if unshare is None: + unshare = unshare_default + if top_level_pidns is None: + top_level_pidns = pidns_default + + logging.info("unet fixture: basename=%s unshare_inline=%s", param, unshare) + _unet = await async_build_topology( + config=get_config(basename=param) if param else None, + rundir=_rundir, + unshare_inline=unshare, + top_level_pidns=top_level_pidns, + pytestconfig=_pytestconfig, + ) + except Exception as error: + logging.debug( + "unet fixture: unet build failed: %s\nparam: %s", + error, + param, + exc_info=True, + ) + pytest.skip( + f"unet fixture: unet build failed: {error}", allow_module_level=True + ) + raise + + try: + tasks = await _unet.run() + except Exception as error: + logging.debug("unet fixture: unet run failed: %s", error, exc_info=True) + await _unet.async_delete() + pytest.skip(f"unet fixture: unet run failed: {error}", allow_module_level=True) + raise + + logging.debug("unet fixture: containers running") + + # Pytest is supposed to always return even if exceptions + try: + yield _unet + except Exception as error: + logging.error("unet fixture: yield unet unexpected exception: %s", error) + + logging.debug("unet fixture: module done, deleting unet") + await _unet.async_delete() + + # No one ever awaits these so cancel them + logging.debug("unet fixture: cleanup") + for task in tasks: + task.cancel() + + # Reset the class variables so auto number is predictable + logging.debug("unet fixture: resetting ords to 1") + L3NodeMixin.next_ord = 1 + Bridge.next_ord = 1 + + +@pytest.fixture(scope="module") +async def unet(request, rundir_module, pytestconfig): # pylint: disable=W0621 + """A unet creating fixutre. + + The request param is either the basename of the config file or a tuple of the form: + (basename, unshare, top_level_pidns), with the second and third elements boolean and + optional, defaulting to False, True. + """ + param = request.param if hasattr(request, "param") else None + sdir = os.path.dirname(os.path.realpath(request.fspath)) + async with achdir(sdir, "unet fixture"): + async for x in _unet_impl(rundir_module, pytestconfig, param=param): + yield x + + +@pytest.fixture(scope="module") +async def unet_share(request, rundir_module, pytestconfig): # pylint: disable=W0621 + """A unet creating fixutre. + + This share variant keeps munet from unsharing the process to a new namespace so that + root level commands and actions are execute on the host, normally they are executed + in the munet namespace which allowing things like scapy inline in tests to work. + + The request param is either the basename of the config file or a tuple of the form: + (basename, top_level_pidns), the second value is a boolean. + """ + param = request.param if hasattr(request, "param") else None + if isinstance(param, (tuple, list)): + param = (param[0], False, param[1]) + sdir = os.path.dirname(os.path.realpath(request.fspath)) + async with achdir(sdir, "unet_share fixture"): + async for x in _unet_impl( + rundir_module, pytestconfig, unshare=False, param=param + ): + yield x + + +@pytest.fixture(scope="module") +async def unet_unshare(request, rundir_module, pytestconfig): # pylint: disable=W0621 + """A unet creating fixutre. + + This unshare variant has the top level munet unshare the process inline so that + root level commands and actions are execute in a new namespace. This allows things + like scapy inline in tests to work. + + The request param is either the basename of the config file or a tuple of the form: + (basename, top_level_pidns), the second value is a boolean. + """ + param = request.param if hasattr(request, "param") else None + if isinstance(param, (tuple, list)): + param = (param[0], True, param[1]) + sdir = os.path.dirname(os.path.realpath(request.fspath)) + async with achdir(sdir, "unet_unshare fixture"): + async for x in _unet_impl( + rundir_module, pytestconfig, unshare=True, param=param + ): + yield x + + +# ================= +# Function Fixtures +# ================= + + +@pytest.fixture(autouse=True, scope="function") +async def function_autouse(request): + async with achdir( + os.path.dirname(os.path.realpath(request.fspath)), "func.fixture" + ): + yield + + +@pytest.fixture(autouse=True) +async def check_for_pause(request, pytestconfig): + # When we unshare inline we can't pause in the pytest_runtest_makereport hook + # so do it here. + if BaseMunet.g_unet and BaseMunet.g_unet.unshare_inline: + pause = bool(pytestconfig.getoption("--pause")) + if pause: + await async_pause_test(f"XXX before test '{request.node.name}'") + yield + + +@pytest.fixture(scope="function") +def stepf(pytestconfig): + class Stepnum: + """Track the stepnum in closure.""" + + num = 0 + + def inc(self): + self.num += 1 + + pause = pytestconfig.getoption("pause") + stepnum = Stepnum() + + def stepfunction(desc=""): + desc = f": {desc}" if desc else "" + if pause: + pause_test(f"before step {stepnum.num}{desc}") + logging.info("STEP %s%s", stepnum.num, desc) + stepnum.inc() + + return stepfunction + + +@pytest_asyncio.fixture(scope="function") +async def astepf(pytestconfig): + class Stepnum: + """Track the stepnum in closure.""" + + num = 0 + + def inc(self): + self.num += 1 + + pause = pytestconfig.getoption("pause") + stepnum = Stepnum() + + async def stepfunction(desc=""): + desc = f": {desc}" if desc else "" + if pause: + await async_pause_test(f"before step {stepnum.num}{desc}") + logging.info("STEP %s%s", stepnum.num, desc) + stepnum.inc() + + return stepfunction + + +@pytest.fixture(scope="function") +def rundir(): + d = os.path.join("/tmp/unet-test", get_test_logdir(module=False)) + logging.debug("conftest: test function rundir %s", d) + return d + + +# Configure logging +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_setup(item): + d = os.path.join( + "/tmp/unet-test", get_test_logdir(nodeid=item.nodeid, module=False) + ) + config = item.config + logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + filename = Path(d, "pytest-exec.log") + logging_plugin.set_log_path(str(filename)) + logging.debug("conftest: test function setup: rundir %s", d) + yield + + +@pytest.fixture +async def unet_perfunc(request, rundir, pytestconfig): # pylint: disable=W0621 + param = request.param if hasattr(request, "param") else None + async for x in _unet_impl(rundir, pytestconfig, param=param): + yield x + + +@pytest.fixture +async def unet_perfunc_unshare(request, rundir, pytestconfig): # pylint: disable=W0621 + """Build unet per test function with an optional topology basename parameter. + + The fixture can be parameterized to choose different config files. + For example, use as follows to run the test with unet_perfunc configured + first with a config file named `cfg1.yaml` then with config file `cfg2.yaml` + (where the actual files could end with `json` or `toml` rather than `yaml`). + + @pytest.mark.parametrize( + "unet_perfunc", ["cfg1", "cfg2]", indirect=["unet_perfunc"] + ) + def test_example(unet_perfunc) + """ + param = request.param if hasattr(request, "param") else None + async for x in _unet_impl(rundir, pytestconfig, unshare=True, param=param): + yield x + + +@pytest.fixture +async def unet_perfunc_share(request, rundir, pytestconfig): # pylint: disable=W0621 + """Build unet per test function with an optional topology basename parameter. + + This share variant keeps munet from unsharing the process to a new namespace so that + root level commands and actions are execute on the host, normally they are executed + in the munet namespace which allowing things like scapy inline in tests to work. + + The fixture can be parameterized to choose different config files. For example, use + as follows to run the test with unet_perfunc configured first with a config file + named `cfg1.yaml` then with config file `cfg2.yaml` (where the actual files could + end with `json` or `toml` rather than `yaml`). + + @pytest.mark.parametrize( + "unet_perfunc", ["cfg1", "cfg2]", indirect=["unet_perfunc"] + ) + def test_example(unet_perfunc) + """ + param = request.param if hasattr(request, "param") else None + async for x in _unet_impl(rundir, pytestconfig, unshare=False, param=param): + yield x diff --git a/tests/topotests/munet/testing/hooks.py b/tests/topotests/munet/testing/hooks.py new file mode 100644 index 0000000..9b6a49a --- /dev/null +++ b/tests/topotests/munet/testing/hooks.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# April 22 2022, Christian Hopps <chopps@gmail.com> +# +# Copyright (c) 2022, LabN Consulting, L.L.C +# +"""A module that implements pytest hooks. + +To use in your project, in your conftest.py add: + + from munet.testing.hooks import * +""" +import logging +import os +import sys +import traceback + +import pytest + +from ..base import BaseMunet # pylint: disable=import-error +from ..cli import cli # pylint: disable=import-error +from .util import pause_test + + +# =================== +# Hooks (non-fixture) +# =================== + + +def pytest_addoption(parser): + parser.addoption( + "--cli-on-error", + action="store_true", + help="CLI on test failure", + ) + + parser.addoption( + "--coverage", + action="store_true", + help="Enable coverage gathering if supported", + ) + + parser.addoption( + "--gdb", + default="", + metavar="HOST[,HOST...]", + help="Comma-separated list of nodes to launch gdb on, or 'all'", + ) + parser.addoption( + "--gdb-breakpoints", + default="", + metavar="BREAKPOINT[,BREAKPOINT...]", + help="Comma-separated list of breakpoints", + ) + parser.addoption( + "--gdb-use-emacs", + action="store_true", + help="Use emacsclient to run gdb instead of a shell", + ) + + parser.addoption( + "--pcap", + default="", + metavar="NET[,NET...]", + help="Comma-separated list of networks to capture packets on, or 'all'", + ) + + parser.addoption( + "--pause", + action="store_true", + help="Pause after each test", + ) + parser.addoption( + "--pause-at-end", + action="store_true", + help="Pause before taking munet down", + ) + parser.addoption( + "--pause-on-error", + action="store_true", + help="Pause after (disables default when --shell or -vtysh given)", + ) + parser.addoption( + "--no-pause-on-error", + dest="pause_on_error", + action="store_false", + help="Do not pause after (disables default when --shell or -vtysh given)", + ) + + parser.addoption( + "--shell", + default="", + metavar="NODE[,NODE...]", + help="Comma-separated list of nodes to spawn shell on, or 'all'", + ) + + parser.addoption( + "--stdout", + default="", + metavar="NODE[,NODE...]", + help="Comma-separated list of nodes to open tail-f stdout window on, or 'all'", + ) + + parser.addoption( + "--stderr", + default="", + metavar="NODE[,NODE...]", + help="Comma-separated list of nodes to open tail-f stderr window on, or 'all'", + ) + + +def pytest_configure(config): + if "PYTEST_XDIST_WORKER" not in os.environ: + os.environ["PYTEST_XDIST_MODE"] = config.getoption("dist", "no") + os.environ["PYTEST_IS_WORKER"] = "" + is_xdist = os.environ["PYTEST_XDIST_MODE"] != "no" + is_worker = False + else: + os.environ["PYTEST_IS_WORKER"] = os.environ["PYTEST_XDIST_WORKER"] + is_xdist = True + is_worker = True + + # Turn on live logging if user specified verbose and the config has a CLI level set + if config.getoption("--verbose") and not is_xdist and not config.getini("log_cli"): + if config.getoption("--log-cli-level", None) is None: + # By setting the CLI option to the ini value it enables log_cli=1 + cli_level = config.getini("log_cli_level") + if cli_level is not None: + config.option.log_cli_level = cli_level + + have_tmux = bool(os.getenv("TMUX", "")) + have_screen = not have_tmux and bool(os.getenv("STY", "")) + have_xterm = not have_tmux and not have_screen and bool(os.getenv("DISPLAY", "")) + have_windows = have_tmux or have_screen or have_xterm + have_windows_pause = have_tmux or have_xterm + xdist_no_windows = is_xdist and not is_worker and not have_windows_pause + + for winopt in ["--shell", "--stdout", "--stderr"]: + b = config.getoption(winopt) + if b and xdist_no_windows: + pytest.exit( + f"{winopt} use requires byobu/TMUX/XTerm " + f"under dist {os.environ['PYTEST_XDIST_MODE']}" + ) + elif b and not is_xdist and not have_windows: + pytest.exit(f"{winopt} use requires byobu/TMUX/SCREEN/XTerm") + + +def pytest_runtest_makereport(item, call): + """Pause or invoke CLI as directed by config.""" + isatty = sys.stdout.isatty() + + pause = bool(item.config.getoption("--pause")) + skipped = False + + if call.excinfo is None: + error = False + elif call.excinfo.typename == "Skipped": + skipped = True + error = False + pause = False + else: + error = True + modname = item.parent.module.__name__ + exval = call.excinfo.value + logging.error( + "test %s/%s failed: %s: stdout: '%s' stderr: '%s'", + modname, + item.name, + exval, + exval.stdout if hasattr(exval, "stdout") else "NA", + exval.stderr if hasattr(exval, "stderr") else "NA", + ) + if not pause: + pause = item.config.getoption("--pause-on-error") + + if error and isatty and item.config.getoption("--cli-on-error"): + if not BaseMunet.g_unet: + logging.error("Could not launch CLI b/c no munet exists yet") + else: + print(f"\nCLI-ON-ERROR: {call.excinfo.typename}") + print(f"CLI-ON-ERROR:\ntest {modname}/{item.name} failed: {exval}") + if hasattr(exval, "stdout") and exval.stdout: + print("stdout: " + exval.stdout.replace("\n", "\nstdout: ")) + if hasattr(exval, "stderr") and exval.stderr: + print("stderr: " + exval.stderr.replace("\n", "\nstderr: ")) + cli(BaseMunet.g_unet) + + if pause: + if skipped: + item.skip_more_pause = True + elif hasattr(item, "skip_more_pause"): + pass + elif call.when == "setup": + if error: + item.skip_more_pause = True + + # we can't asyncio.run() (which pause does) if we are unhsare_inline + # at this point, count on an autouse fixture to pause instead in this + # case + if not BaseMunet.g_unet or not BaseMunet.g_unet.unshare_inline: + pause_test(f"before test '{item.nodeid}'") + + # check for a result to try and catch setup (or module setup) failure + # e.g., after a module level fixture fails, we do not want to pause on every + # skipped test. + elif call.when == "teardown" and call.excinfo: + logging.warning( + "Caught exception during teardown: %s\n:Traceback:\n%s", + call.excinfo, + "".join(traceback.format_tb(call.excinfo.tb)), + ) + pause_test(f"after teardown after test '{item.nodeid}'") + elif call.when == "teardown" and call.result: + pause_test(f"after test '{item.nodeid}'") + elif error: + item.skip_more_pause = True + print(f"\nPAUSE-ON-ERROR: {call.excinfo.typename}") + print(f"PAUSE-ON-ERROR:\ntest {modname}/{item.name} failed: {exval}") + if hasattr(exval, "stdout") and exval.stdout: + print("stdout: " + exval.stdout.replace("\n", "\nstdout: ")) + if hasattr(exval, "stderr") and exval.stderr: + print("stderr: " + exval.stderr.replace("\n", "\nstderr: ")) + pause_test(f"PAUSE-ON-ERROR: '{item.nodeid}'") diff --git a/tests/topotests/munet/testing/util.py b/tests/topotests/munet/testing/util.py new file mode 100644 index 0000000..a1a94bc --- /dev/null +++ b/tests/topotests/munet/testing/util.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# SPDX-License-Identifier: GPL-2.0-or-later +# +# April 22 2022, Christian Hopps <chopps@gmail.com> +# +# Copyright (c) 2022, LabN Consulting, L.L.C +# +"""Utility functions useful when using munet testing functionailty in pytest.""" +import asyncio +import datetime +import functools +import logging +import sys +import time + +from ..base import BaseMunet +from ..cli import async_cli + + +# ================= +# Utility Functions +# ================= + + +async def async_pause_test(desc=""): + isatty = sys.stdout.isatty() + if not isatty: + desc = f" for {desc}" if desc else "" + logging.info("NO PAUSE on non-tty terminal%s", desc) + return + + while True: + if desc: + print(f"\n== PAUSING: {desc} ==") + try: + user = input('PAUSED, "cli" for CLI, "pdb" to debug, "Enter" to continue: ') + except EOFError: + print("^D...continuing") + break + user = user.strip() + if user == "cli": + await async_cli(BaseMunet.g_unet) + elif user == "pdb": + breakpoint() # pylint: disable=W1515 + elif user: + print(f'Unrecognized input: "{user}"') + else: + break + + +def pause_test(desc=""): + asyncio.run(async_pause_test(desc)) + + +def retry(retry_timeout, initial_wait=0, expected=True): + """decorator: retry while functions return is not None or raises an exception. + + * `retry_timeout`: Retry for at least this many seconds; after waiting + initial_wait seconds + * `initial_wait`: Sleeps for this many seconds before first executing function + * `expected`: if False then the return logic is inverted, except for exceptions, + (i.e., a non None ends the retry loop, and returns that value) + """ + + def _retry(func): + @functools.wraps(func) + def func_retry(*args, **kwargs): + retry_sleep = 2 + + # Allow the wrapped function's args to override the fixtures + _retry_timeout = kwargs.pop("retry_timeout", retry_timeout) + _expected = kwargs.pop("expected", expected) + _initial_wait = kwargs.pop("initial_wait", initial_wait) + retry_until = datetime.datetime.now() + datetime.timedelta( + seconds=_retry_timeout + _initial_wait + ) + + if initial_wait > 0: + logging.info("Waiting for [%s]s as initial delay", initial_wait) + time.sleep(initial_wait) + + while True: + seconds_left = (retry_until - datetime.datetime.now()).total_seconds() + try: + ret = func(*args, **kwargs) + if _expected and ret is None: + logging.debug("Function succeeds") + return ret + logging.debug("Function returned %s", ret) + except Exception as error: + logging.info("Function raised exception: %s", str(error)) + ret = error + + if seconds_left < 0: + logging.info("Retry timeout of %ds reached", _retry_timeout) + if isinstance(ret, Exception): + raise ret + return ret + + logging.info( + "Sleeping %ds until next retry with %.1f retry time left", + retry_sleep, + seconds_left, + ) + time.sleep(retry_sleep) + + func_retry._original = func # pylint: disable=W0212 + return func_retry + + return _retry |