path: root/tests/topotests/munet/base.py

# -*- coding: utf-8 eval: (blacken-mode 1) -*-
# SPDX-License-Identifier: GPL-2.0-or-later
#
# July 9 2021, Christian Hopps <chopps@labn.net>
#
# Copyright 2021, LabN Consulting, L.L.C.
#
"""A module that implements core functionality for library or standalone use."""
import asyncio
import datetime
import errno
import ipaddress
import logging
import os
import platform
import re
import readline
import shlex
import signal
import subprocess
import sys
import tempfile
import time as time_mod

from collections import defaultdict
from pathlib import Path
from typing import Union

from . import config as munet_config
from . import linux


try:
    import pexpect

    from pexpect.fdpexpect import fdspawn
    from pexpect.popen_spawn import PopenSpawn

    have_pexpect = True
except ImportError:
    have_pexpect = False

PEXPECT_PROMPT = "PEXPECT_PROMPT>"
PEXPECT_CONTINUATION_PROMPT = "PEXPECT_PROMPT+"

root_hostname = subprocess.check_output("hostname")
our_pid = os.getpid()


detailed_cmd_logging = False


class MunetError(Exception):
    """A generic munet error."""


class CalledProcessError(subprocess.CalledProcessError):
    """Improved logging subclass of subprocess.CalledProcessError."""

    def __str__(self):
        o = self.output.strip() if self.output else ""
        e = self.stderr.strip() if self.stderr else ""
        s = f"returncode: {self.returncode} command: {self.cmd}"
        o = "\n\tstdout: " + o if o else ""
        e = "\n\tstderr: " + e if e else ""
        return s + o + e

    def __repr__(self):
        o = self.output.strip() if self.output else ""
        e = self.stderr.strip() if self.stderr else ""
        return f"munet.base.CalledProcessError({self.returncode}, {self.cmd}, {o}, {e})"


class Timeout:
    """An object to passively monitor for timeouts."""

    def __init__(self, delta):
        self.delta = datetime.timedelta(seconds=delta)
        self.started_on = datetime.datetime.now()
        self.expires_on = self.started_on + self.delta

    def elapsed(self):
        elapsed = datetime.datetime.now() - self.started_on
        return elapsed.total_seconds()

    def is_expired(self):
        return datetime.datetime.now() > self.expires_on

    def remaining(self):
        remaining = self.expires_on - datetime.datetime.now()
        return remaining.total_seconds()

    def __iter__(self):
        return self

    def __next__(self):
        remaining = self.remaining()
        if remaining <= 0:
            raise StopIteration()
        return remaining


def fsafe_name(name):
    return "".join(x if x.isalnum() else "_" for x in name)


def indent(s):
    return "\t" + s.replace("\n", "\n\t")


def shell_quote(command):
    """Return command wrapped in single quotes."""
    if sys.version_info[0] >= 3:
        return shlex.quote(command)
    return "'" + command.replace("'", "'\"'\"'") + "'"


def cmd_error(rc, o, e):
    s = f"rc {rc}"
    o = "\n\tstdout: " + o.strip() if o and o.strip() else ""
    e = "\n\tstderr: " + e.strip() if e and e.strip() else ""
    return s + o + e


def shorten(s):
    s = s.strip()
    i = s.find("\n")
    if i > 0:
        s = s[: i - 1]
        if not s.endswith("..."):
            s += "..."
    if len(s) > 72:
        s = s[:69]
        if not s.endswith("..."):
            s += "..."
    return s


def comm_result(rc, o, e):
    s = f"\n\treturncode {rc}" if rc else ""
    o = "\n\tstdout: " + shorten(o) if o and o.strip() else ""
    e = "\n\tstderr: " + shorten(e) if e and e.strip() else ""
    return s + o + e


def proc_str(p):
    if hasattr(p, "args"):
        args = p.args if isinstance(p.args, str) else " ".join(p.args)
    else:
        args = ""
    return f"proc pid: {p.pid} args: {args}"


def proc_error(p, o, e):
    if hasattr(p, "args"):
        args = p.args if isinstance(p.args, str) else " ".join(p.args)
    else:
        args = ""

    s = f"rc {p.returncode} pid {p.pid}"
    a = "\n\targs: " + args if args else ""
    o = "\n\tstdout: " + (o.strip() if o and o.strip() else "*empty*")
    e = "\n\tstderr: " + (e.strip() if e and e.strip() else "*empty*")
    return s + a + o + e


def comm_error(p):
    rc = p.poll()
    assert rc is not None
    if not hasattr(p, "saved_output"):
        p.saved_output = p.communicate()
    return proc_error(p, *p.saved_output)


async def acomm_error(p):
    rc = p.returncode
    assert rc is not None
    if not hasattr(p, "saved_output"):
        p.saved_output = await p.communicate()
    return proc_error(p, *p.saved_output)


def get_kernel_version():
    kvs = (
        subprocess.check_output("uname -r", shell=True, text=True).strip().split("-", 1)
    )
    kv = kvs[0].split(".")
    kv = [int(x) for x in kv]
    return kv


def convert_number(value) -> int:
    """Convert a number value with a possible suffix to an integer.

    >>> convert_number("100k") == 100 * 1024
    True
    >>> convert_number("100M") == 100 * 1000 * 1000
    True
    >>> convert_number("100Gi") == 100 * 1024 * 1024 * 1024
    True
    >>> convert_number("55") == 55
    True
    """
    if value is None:
        raise ValueError("Invalid value None for convert_number")
    rate = str(value)
    base = 1000
    if rate[-1] == "i":
        base = 1024
        rate = rate[:-1]
    suffix = "KMGTPEZY"
    index = suffix.find(rate[-1])
    if index == -1:
        base = 1024
        index = suffix.lower().find(rate[-1])
    if index != -1:
        rate = rate[:-1]
    return int(rate) * base ** (index + 1)


def is_file_like(fo):
    return isinstance(fo, int) or hasattr(fo, "fileno")


def get_tc_bits_value(user_value):
    value = convert_number(user_value) / 1000
    return f"{value:03f}kbit"


def get_tc_bytes_value(user_value):
    # Raw numbers are bytes in tc
    return convert_number(user_value)


def get_tmp_dir(uniq):
    return os.path.join(tempfile.mkdtemp(), uniq)


async def _async_get_exec_path(binary, cmdf, cache):
    if isinstance(binary, str):
        bins = [binary]
    else:
        bins = binary
    for b in bins:
        if b in cache:
            return cache[b]

        rc, output, _ = await cmdf("which " + b, warn=False)
        if not rc:
            cache[b] = os.path.abspath(output.strip())
            return cache[b]
    return None


def _get_exec_path(binary, cmdf, cache):
    if isinstance(binary, str):
        bins = [binary]
    else:
        bins = binary
    for b in bins:
        if b in cache:
            return cache[b]

        rc, output, _ = cmdf("which " + b, warn=False)
        if not rc:
            cache[b] = os.path.abspath(output.strip())
            return cache[b]
    return None


def get_event_loop():
    """Configure and return our non-thread using event loop.

    This function configures a new child watcher to not use threads.
    Threads cannot be used when we inline unshare a PID namespace.
    """
    policy = asyncio.get_event_loop_policy()
    loop = policy.get_event_loop()
    if not hasattr(os, "pidfd_open"):
        return loop

    owatcher = policy.get_child_watcher()
    logging.debug(
        "event_loop_fixture: global policy %s, current loop %s, current watcher %s",
        policy,
        loop,
        owatcher,
    )

    policy.set_child_watcher(None)
    owatcher.close()

    try:
        watcher = asyncio.PidfdChildWatcher()  # pylint: disable=no-member
    except Exception:
        watcher = asyncio.SafeChildWatcher()
    loop = policy.get_event_loop()

    logging.debug(
        "event_loop_fixture: attaching new watcher %s to loop and setting in policy",
        watcher,
    )
    watcher.attach_loop(loop)
    policy.set_child_watcher(watcher)
    policy.set_event_loop(loop)
    assert asyncio.get_event_loop_policy().get_child_watcher() is watcher

    return loop


class Commander:  # pylint: disable=R0904
    """An object that can execute commands."""

    tmux_wait_gen = 0

    def __init__(self, name, logger=None, unet=None, **kwargs):
        """Create a Commander.

        Args:
            name: name of the commander object
            logger: logger to use for logging commands a defualt is supplied if this
                is None
            unet: unet that owns this object, only used by Commander in run_in_window,
                otherwise can be None.
        """
        # del kwargs  # deal with lint warning
        # logging.warning("Commander: name %s kwargs %s", name, kwargs)

        self.name = name
        self.unet = unet
        self.deleting = False
        self.last = None
        self.exec_paths = {}

        if not logger:
            logname = f"munet.{self.__class__.__name__.lower()}.{name}"
            self.logger = logging.getLogger(logname)
            self.logger.setLevel(logging.DEBUG)
        else:
            self.logger = logger

        super().__init__(**kwargs)

    @property
    def is_vm(self):
        return False

    @property
    def is_container(self):
        return False

    def set_logger(self, logfile):
        self.logger = logging.getLogger(__name__ + ".commander." + self.name)
        self.logger.setLevel(logging.DEBUG)
        if isinstance(logfile, str):
            handler = logging.FileHandler(logfile, mode="w")
        else:
            handler = logging.StreamHandler(logfile)

        fmtstr = "%(asctime)s.%(msecs)03d %(levelname)s: {}({}): %(message)s".format(
            self.__class__.__name__, self.name
        )
        handler.setFormatter(logging.Formatter(fmt=fmtstr))
        self.logger.addHandler(handler)

    def _get_pre_cmd(self, use_str, use_pty, **kwargs):
        """Get the pre-user-command values.

        The values returned here should be what is required to cause the user's command
        to execute in the correct context (e.g., namespace, container, sshremote).
        """
        del kwargs
        del use_pty
        return "" if use_str else []

    def __str__(self):
        return f"{self.__class__.__name__}({self.name})"

    async def async_get_exec_path(self, binary):
        """Return the full path to the binary executable.

        `binary` :: binary name or list of binary names
        """
        return await _async_get_exec_path(
            binary, self.async_cmd_status_nsonly, self.exec_paths
        )

    def get_exec_path(self, binary):
        """Return the full path to the binary executable.

        `binary` :: binary name or list of binary names
        """
        return _get_exec_path(binary, self.cmd_status_nsonly, self.exec_paths)

    def get_exec_path_host(self, binary):
        """Return the full path to the binary executable.

        If the object is actually a derived class (e.g., a container) this method will
        return the exec path for the native namespace rather than the container. The
        path is the one which the other xxx_host methods will use.

        `binary` :: binary name or list of binary names
        """
        return get_exec_path_host(binary)

    def test(self, flags, arg):
        """Run test binary, with flags and arg."""
        test_path = self.get_exec_path(["test"])
        rc, _, _ = self.cmd_status([test_path, flags, arg], warn=False)
        return not rc

    def test_nsonly(self, flags, arg):
        """Run test binary, with flags and arg."""
        test_path = self.get_exec_path(["test"])
        rc, _, _ = self.cmd_status_nsonly([test_path, flags, arg], warn=False)
        return not rc

    def path_exists(self, path):
        """Check if path exists."""
        return self.test("-e", path)

    async def cleanup_pid(self, pid, kill_pid=None):
        """Signal a pid to exit with escalating forcefulness."""
        if kill_pid is None:
            kill_pid = pid

        for sn in (signal.SIGHUP, signal.SIGKILL):
            self.logger.debug(
                "%s: %s %s (wait %s)", self, signal.Signals(sn).name, kill_pid, pid
            )

            os.kill(kill_pid, sn)

            # No need to wait after this.
            if sn == signal.SIGKILL:
                return

            # try each signal, waiting 15 seconds for exit before advancing
            wait_sec = 30
            self.logger.debug("%s: waiting %ss for pid to exit", self, wait_sec)
            for _ in Timeout(wait_sec):
                try:
                    status = os.waitpid(pid, os.WNOHANG)
                    if status == (0, 0):
                        await asyncio.sleep(0.1)
                    else:
                        self.logger.debug("pid %s exited status %s", pid, status)
                        return
                except OSError as error:
                    if error.errno == errno.ECHILD:
                        self.logger.debug("%s: pid %s was reaped", self, pid)
                    else:
                        self.logger.warning(
                            "%s: error waiting on pid %s: %s", self, pid, error
                        )
                    return
            self.logger.debug("%s: timeout waiting on pid %s to exit", self, pid)

    def _get_sub_args(self, cmd_list, defaults, use_pty=False, ns_only=False, **kwargs):
        """Returns pre-command, cmd, and default keyword args."""
        assert not isinstance(cmd_list, str)

        defaults["shell"] = False
        pre_cmd_list = self._get_pre_cmd(False, use_pty, ns_only=ns_only, **kwargs)
        cmd_list = [str(x) for x in cmd_list]

        # os_env = {k: v for k, v in os.environ.items() if k.startswith("MUNET")}
        # env = {**os_env, **(kwargs["env"] if "env" in kwargs else {})}
        env = {**(kwargs["env"] if "env" in kwargs else os.environ)}
        if "MUNET_NODENAME" not in env:
            env["MUNET_NODENAME"] = self.name
        if "MUNET_PID" not in env and "MUNET_PID" in os.environ:
            env["MUNET_PID"] = os.environ["MUNET_PID"]
        kwargs["env"] = env

        defaults.update(kwargs)

        return pre_cmd_list, cmd_list, defaults

    def _common_prologue(self, async_exec, method, cmd, skip_pre_cmd=False, **kwargs):
        cmd_list = self._get_cmd_as_list(cmd)
        if method == "_spawn":
            defaults = {
                "encoding": "utf-8",
                "codec_errors": "ignore",
            }
        else:
            defaults = {
                "stdout": subprocess.PIPE,
                "stderr": subprocess.PIPE,
            }
            if not async_exec:
                defaults["encoding"] = "utf-8"

        pre_cmd_list, cmd_list, defaults = self._get_sub_args(
            cmd_list, defaults, **kwargs
        )

        use_pty = kwargs.get("use_pty", False)
        if method == "_spawn":
            # spawn doesn't take "shell" keyword arg
            if "shell" in defaults:
                del defaults["shell"]
            # this is required to avoid receiving a STOPPED signal on expect!
            if not use_pty:
                defaults["preexec_fn"] = os.setsid
            defaults["env"]["PS1"] = "$ "

        if not detailed_cmd_logging:
            pre_cmd_str = shlex.join(pre_cmd_list) if not skip_pre_cmd else ""
            if "nsenter" in pre_cmd_str:
                self.logger.debug('%s("%s")', method, shlex.join(cmd_list))
            elif pre_cmd_str:
                self.logger.debug(
                    '%s("%s") [precmd: %s]', method, shlex.join(cmd_list), pre_cmd_str
                )
            else:
                self.logger.debug('%s("%s") [no precmd]', method, shlex.join(cmd_list))
        else:
            self.logger.debug(
                '%s: %s("%s", pre_cmd: "%s" use_pty: %s kwargs: %.120s)',
                self,
                method,
                cmd_list,
                pre_cmd_list if not skip_pre_cmd else "",
                use_pty,
                defaults,
            )

        actual_cmd_list = cmd_list if skip_pre_cmd else pre_cmd_list + cmd_list
        return actual_cmd_list, defaults

    async def _async_popen(self, method, cmd, **kwargs):
        """Create a new asynchronous subprocess."""
        acmd, kwargs = self._common_prologue(True, method, cmd, **kwargs)
        p = await asyncio.create_subprocess_exec(*acmd, **kwargs)
        return p, acmd

    def _popen(self, method, cmd, **kwargs):
        """Create a subprocess."""
        acmd, kwargs = self._common_prologue(False, method, cmd, **kwargs)
        p = subprocess.Popen(acmd, **kwargs)
        return p, acmd

    def _fdspawn(self, fo, **kwargs):
        defaults = {}
        defaults.update(kwargs)

        if "echo" in defaults:
            del defaults["echo"]

        if "encoding" not in defaults:
            defaults["encoding"] = "utf-8"
            if "codec_errors" not in defaults:
                defaults["codec_errors"] = "ignore"
        encoding = defaults["encoding"]

        self.logger.debug("%s: _fdspawn(%s, kwargs: %s)", self, fo, defaults)

        p = fdspawn(fo, **defaults)

        # We don't have TTY like conversions of LF to CRLF
        p.crlf = os.linesep.encode(encoding)

        # we own the socket now detach the file descriptor to keep it from closing
        if hasattr(fo, "detach"):
            fo.detach()

        return p

    def _spawn(self, cmd, skip_pre_cmd=False, use_pty=False, echo=False, **kwargs):
        logging.debug(
            '%s: _spawn: cmd "%s" skip_pre_cmd %s use_pty %s echo %s kwargs %s',
            self,
            cmd,
            skip_pre_cmd,
            use_pty,
            echo,
            kwargs,
        )
        actual_cmd, defaults = self._common_prologue(
            False, "_spawn", cmd, skip_pre_cmd=skip_pre_cmd, use_pty=use_pty, **kwargs
        )

        self.logger.debug(
            '%s: %s("%s", use_pty %s echo %s defaults: %s)',
            self,
            "PopenSpawn" if not use_pty else "pexpect.spawn",
            actual_cmd,
            use_pty,
            echo,
            defaults,
        )

        # We don't specify a timeout it defaults to 30s is that OK?
        if not use_pty:
            p = PopenSpawn(actual_cmd, **defaults)
        else:
            p = pexpect.spawn(actual_cmd[0], actual_cmd[1:], echo=echo, **defaults)
        return p, actual_cmd

    def spawn(
        self,
        cmd,
        spawned_re,
        expects=(),
        sends=(),
        use_pty=False,
        logfile=None,
        logfile_read=None,
        logfile_send=None,
        trace=None,
        **kwargs,
    ):
        """Create a spawned send/expect process.

        Args:
            cmd: list of args to exec/popen with, or an already open socket
            spawned_re: what to look for to know when done, `spawn` returns when seen
            expects: a list of regex other than `spawned_re` to look for. Commonly,
                "ogin:" or "[Pp]assword:"r.
            sends: what to send when an element of `expects` matches. So e.g., the
                username or password if thats what corresponding expect matched. Can
                be the empty string to send nothing.
            use_pty: true for pty based expect, otherwise uses popen (pipes/files)
            trace: if true then log send/expects
            **kwargs - kwargs passed on the _spawn.

        Returns:
            A pexpect process.

        Raises:
            pexpect.TIMEOUT, pexpect.EOF as documented in `pexpect`
            CalledProcessError if EOF is seen and `cmd` exited then
                raises a CalledProcessError to indicate the failure.
        """
        if is_file_like(cmd):
            assert not use_pty
            ac = "*socket*"
            p = self._fdspawn(cmd, **kwargs)
        else:
            p, ac = self._spawn(cmd, use_pty=use_pty, **kwargs)

        if logfile:
            p.logfile = logfile
        if logfile_read:
            p.logfile_read = logfile_read
        if logfile_send:
            p.logfile_send = logfile_send

        # for spawned shells (i.e., a direct command an not a console)
        # this is wrong and will cause 2 prompts
        if not use_pty:
            # This isn't very nice looking
            p.echo = False
            if not is_file_like(cmd):
                p.isalive = lambda: p.proc.poll() is None
            if not hasattr(p, "close"):
                p.close = p.wait

        # Do a quick check to see if we got the prompt right away, otherwise we may be
        # at a console so we send a \n to re-issue the prompt
        index = p.expect([spawned_re, pexpect.TIMEOUT, pexpect.EOF], timeout=0.1)
        if index == 0:
            assert p.match is not None
            self.logger.debug(
                "%s: got spawned_re quick: '%s' matching '%s'",
                self,
                p.match.group(0),
                spawned_re,
            )
            return p

        # Now send a CRLF to cause the prompt (or whatever else) to re-issue
        p.send("\n")
        try:
            patterns = [spawned_re, *expects]

            self.logger.debug("%s: expecting: %s", self, patterns)

            while index := p.expect(patterns):
                if trace:
                    assert p.match is not None
                    self.logger.debug(
                        "%s: got expect: '%s' matching %d '%s', sending '%s'",
                        self,
                        p.match.group(0),
                        index,
                        patterns[index],
                        sends[index - 1],
                    )
                if sends[index - 1]:
                    p.send(sends[index - 1])

                self.logger.debug("%s: expecting again: %s", self, patterns)
            self.logger.debug(
                "%s: got spawned_re: '%s' matching '%s'",
                self,
                p.match.group(0),
                spawned_re,
            )
            return p
        except pexpect.TIMEOUT:
            self.logger.error(
                "%s: TIMEOUT looking for spawned_re '%s' expect buffer so far:\n%s",
                self,
                spawned_re,
                indent(p.buffer),
            )
            raise
        except pexpect.EOF as eoferr:
            if p.isalive():
                raise
            rc = p.status
            before = indent(p.before)
            error = CalledProcessError(rc, ac, output=before)
            self.logger.error(
                "%s: EOF looking for spawned_re '%s' before EOF:\n%s",
                self,
                spawned_re,
                before,
            )
            p.close()
            raise error from eoferr

    async def shell_spawn(
        self,
        cmd,
        prompt,
        expects=(),
        sends=(),
        use_pty=False,
        will_echo=False,
        is_bourne=True,
        init_newline=False,
        **kwargs,
    ):
        """Create a shell REPL (read-eval-print-loop).

        Args:
            cmd: shell and list of args to popen with, or an already open socket
            prompt: the REPL prompt to look for, the function returns when seen
            expects: a list of regex other than `spawned_re` to look for. Commonly,
                "ogin:" or "[Pp]assword:"r.
            sends: what to send when an element of `expects` matches. So e.g., the
                username or password if thats what corresponding expect matched. Can
                be the empty string to send nothing.
            is_bourne: if False then do not modify shell prompt for internal
                parser friently format, and do not expect continuation prompts.
            init_newline: send an initial newline for non-bourne shell spawns, otherwise
                expect the prompt simply from running the command
            use_pty: true for pty based expect, otherwise uses popen (pipes/files)
            will_echo: bash is buggy in that it echo's to non-tty unlike any other
                sh/ksh, set this value to true if running back
            **kwargs - kwargs passed on the _spawn.
        """
        combined_prompt = r"({}|{})".format(re.escape(PEXPECT_PROMPT), prompt)

        assert not is_file_like(cmd) or not use_pty
        p = self.spawn(
            cmd,
            combined_prompt,
            expects=expects,
            sends=sends,
            use_pty=use_pty,
            echo=False,
            **kwargs,
        )
        assert not p.echo

        if not is_bourne:
            if init_newline:
                p.send("\n")
            return ShellWrapper(p, prompt, will_echo=will_echo)

        ps1 = PEXPECT_PROMPT
        ps2 = PEXPECT_CONTINUATION_PROMPT

        # Avoid problems when =/usr/bin/env= prints the values
        ps1p = ps1[:5] + "${UNSET_V}" + ps1[5:]
        ps2p = ps2[:5] + "${UNSET_V}" + ps2[5:]

        ps1 = re.escape(ps1)
        ps2 = re.escape(ps2)
        extra = [
            "TERM=dumb",
            "set +o emacs",
            "set +o vi",
            "unset HISTFILE",
            "PAGER=cat",
            "export PAGER",
        ]
        pchg = "PS1='{0}' PS2='{1}' PROMPT_COMMAND=''\n".format(ps1p, ps2p)
        p.send(pchg)
        return ShellWrapper(p, ps1, ps2, extra_init_cmd=extra, will_echo=will_echo)

    def popen(self, cmd, **kwargs):
        """Creates a pipe with the given `command`.

        Args:
            cmd: `str` or `list` of command to open a pipe with.
            **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
                then will be invoked with `bash -c`, otherwise `command` is a list and
                will be invoked without a shell.

        Returns:
            a subprocess.Popen object.
        """
        return self._popen("popen", cmd, **kwargs)[0]

    def popen_nsonly(self, cmd, **kwargs):
        """Creates a pipe with the given `command`.

        Args:
            cmd: `str` or `list` of command to open a pipe with.
            **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
                then will be invoked with `bash -c`, otherwise `command` is a list and
                will be invoked without a shell.

        Returns:
            a subprocess.Popen object.
        """
        return self._popen("popen_nsonly", cmd, ns_only=True, **kwargs)[0]

    async def async_popen(self, cmd, **kwargs):
        """Creates a pipe with the given `command`.

        Args:
            cmd: `str` or `list` of command to open a pipe with.
            **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
                `command` is a string then will be invoked with `bash -c`, otherwise
                `command` is a list and will be invoked without a shell.

        Returns:
            a asyncio.subprocess.Process object.
        """
        p, _ = await self._async_popen("async_popen", cmd, **kwargs)
        return p

    async def async_popen_nsonly(self, cmd, **kwargs):
        """Creates a pipe with the given `command`.

        Args:
            cmd: `str` or `list` of command to open a pipe with.
            **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
                `command` is a string then will be invoked with `bash -c`, otherwise
                `command` is a list and will be invoked without a shell.

        Returns:
            a asyncio.subprocess.Process object.
        """
        p, _ = await self._async_popen(
            "async_popen_nsonly", cmd, ns_only=True, **kwargs
        )
        return p

    async def async_cleanup_proc(self, p, pid=None):
        """Terminate a process started with a popen call.

        Args:
            p: return value from :py:`async_popen`, :py:`popen`, et al.
            pid: pid to signal instead of p.pid, typically a child of
                cmd_p == nsenter.

        Returns:
            None on success, the ``p`` if multiple timeouts occur even
            after a SIGKILL sent.
        """
        if not p:
            return None

        if p.returncode is not None:
            if isinstance(p, subprocess.Popen):
                o, e = p.communicate()
            else:
                o, e = await p.communicate()
            self.logger.debug(
                "%s: [cleanup_proc] proc already exited status: %s",
                self,
                proc_error(p, o, e),
            )
            return None

        if pid is None:
            pid = p.pid

        self.logger.debug(
            "%s: [cleanup_proc] terminate process: %s (pid %s)", self, proc_str(p), pid
        )
        try:
            # This will SIGHUP and wait a while then SIGKILL and return immediately
            await self.cleanup_pid(p.pid, pid)

            # Wait another 2 seconds after the possible SIGKILL above for the
            # parent nsenter to cleanup and exit
            wait_secs = 2
            if isinstance(p, subprocess.Popen):
                o, e = p.communicate(timeout=wait_secs)
            else:
                o, e = await asyncio.wait_for(p.communicate(), timeout=wait_secs)
            self.logger.debug(
                "%s: [cleanup_proc] exited after kill, status: %s",
                self,
                proc_error(p, o, e),
            )
        except (asyncio.TimeoutError, subprocess.TimeoutExpired):
            self.logger.warning("%s: [cleanup_proc] SIGKILL timeout", self)
            return p
        except Exception as error:
            self.logger.warning(
                "%s: [cleanup_proc] kill unexpected exception: %s",
                self,
                error,
                exc_info=True,
            )
            return p
        return None

    @staticmethod
    def _cmd_status_input(stdin):
        pinput = None
        if isinstance(stdin, (bytes, str)):
            pinput = stdin
            stdin = subprocess.PIPE
        return pinput, stdin

    def _cmd_status_finish(self, p, c, ac, o, e, raises, warn):
        rc = p.returncode
        self.last = (rc, ac, c, o, e)
        if not rc:
            resstr = comm_result(rc, o, e)
            if resstr:
                self.logger.debug("%s", resstr)
        else:
            if warn:
                self.logger.warning("%s: proc failed: %s", self, proc_error(p, o, e))
            if raises:
                # error = Exception("stderr: {}".format(stderr))
                # This annoyingly doesnt' show stderr when printed normally
                raise CalledProcessError(rc, ac, o, e)
        return rc, o, e

    def _cmd_status(self, cmds, raises=False, warn=True, stdin=None, **kwargs):
        """Execute a command."""
        timeout = None
        if "timeout" in kwargs:
            timeout = kwargs["timeout"]
            del kwargs["timeout"]

        pinput, stdin = Commander._cmd_status_input(stdin)
        p, actual_cmd = self._popen("cmd_status", cmds, stdin=stdin, **kwargs)
        o, e = p.communicate(pinput, timeout=timeout)
        return self._cmd_status_finish(p, cmds, actual_cmd, o, e, raises, warn)

    async def _async_cmd_status(
        self, cmds, raises=False, warn=True, stdin=None, text=None, **kwargs
    ):
        """Execute a command."""
        timeout = None
        if "timeout" in kwargs:
            timeout = kwargs["timeout"]
            del kwargs["timeout"]

        pinput, stdin = Commander._cmd_status_input(stdin)
        p, actual_cmd = await self._async_popen(
            "async_cmd_status", cmds, stdin=stdin, **kwargs
        )

        if text is False:
            encoding = None
        else:
            encoding = kwargs.get("encoding", "utf-8")

        if encoding is not None and isinstance(pinput, str):
            pinput = pinput.encode(encoding)
        try:
            o, e = await asyncio.wait_for(p.communicate(), timeout=timeout)
        except (TimeoutError, asyncio.TimeoutError) as error:
            raise subprocess.TimeoutExpired(
                cmd=actual_cmd, timeout=timeout, output=None, stderr=None
            ) from error
        if encoding is not None:
            o = o.decode(encoding) if o is not None else o
            e = e.decode(encoding) if e is not None else e
        return self._cmd_status_finish(p, cmds, actual_cmd, o, e, raises, warn)

    def _get_cmd_as_list(self, cmd):
        """Given a list or string return a list form for execution.

        If `cmd` is a string then the returned list uses bash and looks
        like this: ["/bin/bash", "-c", cmd]. Some node types override
        this function if they utilize a different shell as to return
        a different list of values.

        Args:
            cmd: list or string representing the command to execute.

        Returns:
            list of commands to execute.
        """
        if not isinstance(cmd, str):
            cmds = cmd
        else:
            # Make sure the code doesn't think `cd` will work.
            assert not re.match(r"cd(\s*|\s+(\S+))$", cmd)
            cmds = ["/bin/bash", "-c", cmd]
        return cmds

    def cmd_nostatus(self, cmd, **kwargs):
        """Run given command returning output[s].

        Args:
            cmd: `str` or `list` of the command to execute.  If a string is given
                it is run using a shell, otherwise the list is executed directly
                as the binary and arguments.
            **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
                then will be invoked with `bash -c`, otherwise `command` is a list and
                will be invoked without a shell.

        Returns:
            if "stderr" is in kwargs and not equal to subprocess.STDOUT, then
            both stdout and stderr are returned, otherwise stderr is combined
            with stdout and only stdout is returned.
        """
        #
        # This method serves as the basis for all derived sync cmd variations, so to
        # override sync cmd behavior simply override this function and *not* the other
        # variations, unless you are changing only that variation's behavior
        #

        # XXX change this back to _cmd_status instead of cmd_status when we
        # consolidate and cleanup the container overrides of *cmd_* functions

        cmds = cmd
        if "stderr" in kwargs and kwargs["stderr"] != subprocess.STDOUT:
            _, o, e = self.cmd_status(cmds, **kwargs)
            return o, e
        if "stderr" in kwargs:
            del kwargs["stderr"]
        _, o, _ = self.cmd_status(cmds, stderr=subprocess.STDOUT, **kwargs)
        return o

    def cmd_status(self, cmd, **kwargs):
        """Run given command returning status and outputs.

        Args:
            cmd: `str` or `list` of the command to execute.  If a string is given
                it is run using a shell, otherwise the list is executed directly
                as the binary and arguments.
            **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
                then will be invoked with `bash -c`, otherwise `command` is a list and
                will be invoked without a shell.

        Returns:
            (status, output, error) are returned
            status: the returncode of the command.
            output: stdout as a string from the command.
            error: stderr as a string from the command.
        """
        #
        # This method serves as the basis for all derived sync cmd variations, so to
        # override sync cmd behavior simply override this function and *not* the other
        # variations, unless you are changing only that variation's behavior
        #
        return self._cmd_status(cmd, **kwargs)

    def cmd_raises(self, cmd, **kwargs):
        """Execute a command. Raise an exception on errors.

        Args:
            cmd: `str` or `list` of the command to execute.  If a string is given
                it is run using a shell, otherwise the list is executed directly
                as the binary and arguments.
            **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
                then will be invoked with `bash -c`, otherwise `command` is a list and
                will be invoked without a shell.

        Returns:
            output: stdout as a string from the command.

        Raises:
            CalledProcessError: on non-zero exit status
        """
        _, stdout, _ = self._cmd_status(cmd, raises=True, **kwargs)
        return stdout

    def cmd_nostatus_nsonly(self, cmd, **kwargs):
        # Make sure the command runs on the host and not in any container.
        return self.cmd_nostatus(cmd, ns_only=True, **kwargs)

    def cmd_status_nsonly(self, cmd, **kwargs):
        # Make sure the command runs on the host and not in any container.
        return self._cmd_status(cmd, ns_only=True, **kwargs)

    def cmd_raises_nsonly(self, cmd, **kwargs):
        # Make sure the command runs on the host and not in any container.
        _, stdout, _ = self._cmd_status(cmd, raises=True, ns_only=True, **kwargs)
        return stdout

    async def async_cmd_status(self, cmd, **kwargs):
        """Run given command returning status and outputs.

        Args:
            cmd: `str` or `list` of the command to execute.  If a string is given
                it is run using a shell, otherwise the list is executed directly
                as the binary and arguments.
            **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
                `cmd` is a string then will be invoked with `bash -c`, otherwise
                `cmd` is a list and will be invoked without a shell.

        Returns:
            (status, output, error) are returned
            status: the returncode of the command.
            output: stdout as a string from the command.
            error: stderr as a string from the command.
        """
        #
        # This method serves as the basis for all derived async cmd variations, so to
        # override async cmd behavior simply override this function and *not* the other
        # variations, unless you are changing only that variation's behavior
        #
        return await self._async_cmd_status(cmd, **kwargs)

    async def async_cmd_nostatus(self, cmd, **kwargs):
        """Run given command returning output[s].

        Args:
            cmd: `str` or `list` of the command to execute.  If a string is given
                it is run using a shell, otherwise the list is executed directly
                as the binary and arguments.
            **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
                `cmd` is a string then will be invoked with `bash -c`, otherwise
                `cmd` is a list and will be invoked without a shell.

        Returns:
            if "stderr" is in kwargs and not equal to subprocess.STDOUT, then
            both stdout and stderr are returned, otherwise stderr is combined
            with stdout and only stdout is returned.

        """
        # XXX change this back to _async_cmd_status instead of cmd_status when we
        # consolidate and cleanup the container overrides of *cmd_* functions

        cmds = cmd
        if "stderr" in kwargs and kwargs["stderr"] != subprocess.STDOUT:
            _, o, e = await self._async_cmd_status(cmds, **kwargs)
            return o, e
        if "stderr" in kwargs:
            del kwargs["stderr"]
        _, o, _ = await self._async_cmd_status(cmds, stderr=subprocess.STDOUT, **kwargs)
        return o

    async def async_cmd_raises(self, cmd, **kwargs):
        """Execute a command. Raise an exception on errors.

        Args:
            cmd: `str` or `list` of the command to execute.  If a string is given
                it is run using a shell, otherwise the list is executed directly
                as the binary and arguments.
            **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
                `cmd` is a string then will be invoked with `bash -c`, otherwise
                `cmd` is a list and will be invoked without a shell.

        Returns:
            output: stdout as a string from the command.

        Raises:
            CalledProcessError: on non-zero exit status
        """
        _, stdout, _ = await self._async_cmd_status(cmd, raises=True, **kwargs)
        return stdout

    async def async_cmd_status_nsonly(self, cmd, **kwargs):
        # Make sure the command runs on the host and not in any container.
        return await self._async_cmd_status(cmd, ns_only=True, **kwargs)

    async def async_cmd_raises_nsonly(self, cmd, **kwargs):
        # Make sure the command runs on the host and not in any container.
        _, stdout, _ = await self._async_cmd_status(
            cmd, raises=True, ns_only=True, **kwargs
        )
        return stdout

    def cmd_legacy(self, cmd, **kwargs):
        """Execute a command with stdout and stderr joined, *IGNORES ERROR*."""
        defaults = {"stderr": subprocess.STDOUT}
        defaults.update(kwargs)
        _, stdout, _ = self._cmd_status(cmd, raises=False, **defaults)
        return stdout

    # Run a command in a new window (gnome-terminal, screen, tmux, xterm)
    def run_in_window(
        self,
        cmd,
        wait_for=False,
        background=False,
        name=None,
        title=None,
        forcex=False,
        new_window=False,
        tmux_target=None,
        ns_only=False,
    ):
        """Run a command in a new window (TMUX, Screen or XTerm).

        Args:
            cmd: string to execute.
            wait_for: True to wait for exit from command or `str` as channel neme to
                signal on exit, otherwise False
            background: Do not change focus to new window.
            title: Title for new pane (tmux) or window (xterm).
            name: Name of the new window (tmux)
            forcex: Force use of X11.
            new_window: Open new window (instead of pane) in TMUX
            tmux_target: Target for tmux pane.

        Returns:
            the pane/window identifier from TMUX (depends on `new_window`)
        """
        channel = None
        if isinstance(wait_for, str):
            channel = wait_for
        elif wait_for is True:
            channel = "{}-wait-{}".format(our_pid, Commander.tmux_wait_gen)
            Commander.tmux_wait_gen += 1

        if forcex or ("TMUX" not in os.environ and "STY" not in os.environ):
            root_level = False
        else:
            root_level = True

        # SUDO: The important thing to note is that with all these methods we are
        # executing on the users windowing system, so even though we are normally
        # running as root, we will not be when the command is dispatched. Also
        # in the case of SCREEN and X11 we need to sudo *back* to the user as well
        # This is also done by SSHRemote by defualt so we should *not* sudo back
        # if we are SSHRemote.

        # XXX need to test ssh in screen
        # XXX need to test ssh in Xterm
        sudo_path = get_exec_path_host(["sudo"])
        # This first test case seems same as last but using list instead of string?
        if self.is_vm and self.use_ssh and not ns_only:  # pylint: disable=E1101
            if isinstance(cmd, str):
                cmd = shlex.split(cmd)
            cmd = [
                "/usr/bin/env",
                f"MUNET_NODENAME={self.name}",
            ]
            if "MUNET_PID" in os.environ:
                cmd.append(f"MUNET_PID={os.environ.get('MUNET_PID')}")
            cmd += cmd

            # get the ssh cmd
            cmd = self._get_pre_cmd(False, True, ns_only=ns_only) + [shlex.join(cmd)]
            unet = self.unet  # pylint: disable=E1101
            uns_cmd = unet._get_pre_cmd(  # pylint: disable=W0212
                False, True, ns_only=True, root_level=root_level
            )
            # get the nsenter for munet
            nscmd = [
                sudo_path,
                *uns_cmd,
                *cmd,
            ]
        else:
            # This is the command to execute to be inside the namespace.
            # We are getting into trouble with quoting.
            envvars = f"MUNET_NODENAME={self.name} NODENAME={self.name}"
            if hasattr(self, "rundir"):
                envvars += f" RUNDIR={self.rundir}"
            if "MUNET_PID" in os.environ:
                envvars += f" MUNET_PID={os.environ.get('MUNET_PID')}"
            if hasattr(self.unet, "config_dirname") and self.unet.config_dirname:
                envvars += f" CONFIGDIR={self.unet.config_dirname}"
            elif "CONFIGDIR" in os.environ:
                envvars += f" CONFIGDIR={os.environ['CONFIGDIR']}"
            cmd = f"/usr/bin/env {envvars} {cmd}"
            # We need sudo b/c we are executing as the user inside the window system.
            sudo_path = get_exec_path_host(["sudo"])
            nscmd = (
                sudo_path
                + " "
                + self._get_pre_cmd(True, True, ns_only=ns_only, root_level=root_level)
                + " "
                + cmd
            )

        if "TMUX" in os.environ and not forcex:
            cmd = [get_exec_path_host("tmux")]
            if new_window:
                cmd.append("new-window")
                cmd.append("-P")
                if name:
                    cmd.append("-n")
                    cmd.append(name)
                if tmux_target:
                    cmd.append("-t")
                    cmd.append(tmux_target)
            else:
                cmd.append("split-window")
                cmd.append("-P")
                cmd.append("-h")
                if not tmux_target:
                    tmux_target = os.getenv("TMUX_PANE", "")
            if background:
                cmd.append("-d")
            if tmux_target:
                cmd.append("-t")
                cmd.append(tmux_target)

            # nscmd is always added as single string argument
            if not isinstance(nscmd, str):
                nscmd = shlex.join(nscmd)
            if title:
                nscmd = f"printf '\033]2;{title}\033\\'; {nscmd}"
            if channel:
                nscmd = f'trap "tmux wait -S {channel}; exit 0" EXIT; {nscmd}'
            cmd.append(nscmd)

        elif "STY" in os.environ and not forcex:
            # wait for not supported in screen for now
            channel = None
            cmd = [get_exec_path_host("screen")]
            if not os.path.exists(
                "/run/screen/S-{}/{}".format(os.environ["USER"], os.environ["STY"])
            ):
                # XXX not appropriate for ssh
                cmd = ["sudo", "-Eu", os.environ["SUDO_USER"]] + cmd

            if title:
                cmd.append("-t")
                cmd.append(title)

            if isinstance(nscmd, str):
                nscmd = shlex.split(nscmd)
            cmd.extend(nscmd)
        elif "DISPLAY" in os.environ:
            cmd = [get_exec_path_host("xterm")]
            if "SUDO_USER" in os.environ:
                # Do this b/c making things work as root with xauth seems hard
                cmd = [
                    get_exec_path_host("sudo"),
                    "-Eu",
                    os.environ["SUDO_USER"],
                ] + cmd
            if title:
                cmd.append("-T")
                cmd.append(title)

            cmd.append("-e")
            if isinstance(nscmd, str):
                cmd.extend(shlex.split(nscmd))
            else:
                cmd.extend(nscmd)

            # if channel:
            #    return self.cmd_raises(cmd, skip_pre_cmd=True)
            # else:
            p = commander.popen(
                cmd,
                # skip_pre_cmd=True,
                stdin=None,
                shell=False,
            )
            # We should reap the child and report the error then.
            time_mod.sleep(2)
            if p.poll() is not None:
                self.logger.error("%s: Failed to launch xterm: %s", self, comm_error(p))
            return p
        else:
            self.logger.error(
                "DISPLAY, STY, and TMUX not in environment, can't open window"
            )
            raise Exception("Window requestd but TMUX, Screen and X11 not available")

        # pane_info = self.cmd_raises(cmd, skip_pre_cmd=True, ns_only=True).strip()
        # We are prepending the nsenter command, so use unet.rootcmd
        pane_info = commander.cmd_raises(cmd).strip()

        # Re-adjust the layout
        if "TMUX" in os.environ:
            cmd = [
                get_exec_path_host("tmux"),
                "select-layout",
                "-t",
                pane_info if not tmux_target else tmux_target,
                "even-horizontal",
            ]
            commander.cmd_status(cmd)
            cmd = [
                get_exec_path_host("tmux"),
                "select-layout",
                "-t",
                pane_info if not tmux_target else tmux_target,
                "tiled",
            ]
            commander.cmd_status(cmd)

        # Wait here if we weren't handed the channel to wait for
        if channel and wait_for is True:
            cmd = [get_exec_path_host("tmux"), "wait", channel]
            # commander.cmd_status(cmd, skip_pre_cmd=True)
            commander.cmd_status(cmd)

        return pane_info

    def delete(self):
        """Calls self.async_delete within an exec loop."""
        asyncio.run(self.async_delete())

    async def _async_delete(self):
        """Delete this objects resources.

        This is the actual implementation of the resource cleanup, each class
        should cleanup it's own resources, generally catching and reporting,
        but not reraising any exceptions for it's own cleanup, then it should
        invoke `super()._async_delete() without catching any exceptions raised
        therein. See other examples in `base.py` or `native.py`
        """
        self.logger.info("%s: deleted", self)

    async def async_delete(self):
        """Delete the Commander (or derived object).

        The actual implementation for any class should be in `_async_delete`
        new derived classes should look at the documentation for that function.
        """
        try:
            self.deleting = True
            await self._async_delete()
        except Exception as error:
            self.logger.error("%s: error while deleting: %s", self, error)


class InterfaceMixin:
    """A mixin class to support interface functionality."""

    def __init__(self, *args, **kwargs):
        # del kwargs  # get rid of lint
        # logging.warning("InterfaceMixin: args: %s kwargs: %s", args, kwargs)

        self._intf_addrs = defaultdict(lambda: [None, None])
        self.net_intfs = {}
        self.next_intf_index = 0
        self.basename = "eth"
        # self.basename = name + "-eth"
        super().__init__(*args, **kwargs)

    @property
    def intfs(self):
        return sorted(self._intf_addrs.keys())

    @property
    def networks(self):
        return sorted(self.net_intfs.keys())

    def get_intf_addr(self, ifname, ipv6=False):
        if ifname not in self._intf_addrs:
            return None
        return self._intf_addrs[ifname][bool(ipv6)]

    def set_intf_addr(self, ifname, ifaddr):
        ifaddr = ipaddress.ip_interface(ifaddr)
        self._intf_addrs[ifname][ifaddr.version == 6] = ifaddr

    def net_addr(self, netname, ipv6=False):
        if netname not in self.net_intfs:
            return None
        return self.get_intf_addr(self.net_intfs[netname], ipv6=ipv6)

    def set_intf_basename(self, basename):
        self.basename = basename

    def get_next_intf_name(self):
        while True:
            ifname = self.basename + str(self.next_intf_index)
            self.next_intf_index += 1
            if ifname not in self._intf_addrs:
                break
        return ifname

    def get_ns_ifname(self, ifname):
        """Return a namespace unique interface name.

        This function is primarily overriden by L3QemuVM, IOW by any class
        that doesn't create it's own network namespace and will share that
        with the root (unet) namespace.

        Args:
            ifname: the interface name.

        Returns:
            A name unique to the namespace of this object. By defualt the assumption
            is the ifname is namespace unique.
        """
        return ifname

    def register_interface(self, ifname):
        if ifname not in self._intf_addrs:
            self._intf_addrs[ifname] = [None, None]

    def register_network(self, netname, ifname):
        if netname in self.net_intfs:
            assert self.net_intfs[netname] == ifname
        else:
            self.net_intfs[netname] = ifname

    def get_linux_tc_args(self, ifname, config):
        """Get interface constraints (jitter, delay, rate) for linux TC.

        The keys and their values are as follows:

        delay (int): number of microseconds
        jitter (int): number of microseconds
        jitter-correlation (float): % correlation to previous (default 10%)
        loss (float): % of loss
        loss-correlation (float): % correlation to previous (default 0%)
        rate  (int or str): bits per second, string allows for use of
            {KMGTKiMiGiTi} prefixes "i" means K == 1024 otherwise K == 1000
        """
        del ifname  # unused

        netem_args = ""

        def get_number(c, v, d=None):
            if v not in c or c[v] is None:
                return d
            return convert_number(c[v])

        delay = get_number(config, "delay")
        if delay is not None:
            netem_args += f" delay {delay}usec"

        jitter = get_number(config, "jitter")
        if jitter is not None:
            if not delay:
                raise ValueError("jitter but no delay specified")
            jitter_correlation = get_number(config, "jitter-correlation", 10)
            netem_args += f" {jitter}usec {jitter_correlation}%"

        loss = get_number(config, "loss")
        if loss is not None:
            loss_correlation = get_number(config, "loss-correlation", 0)
            if loss_correlation:
                netem_args += f" loss {loss}% {loss_correlation}%"
            else:
                netem_args += f" loss {loss}%"

        if (o_rate := config.get("rate")) is None:
            return netem_args, ""

        #
        # This comment is not correct, but is trying to talk through/learn the
        # machinery.
        #
        # tokens arrive at `rate` into token buffer.
        # limit - number of bytes that can be queued waiting for tokens
        #   -or-
        # latency - maximum amount of time a packet may sit in TBF queue
        #
        # So this just allows receiving faster than rate for latency amount of
        # time, before dropping.
        #
        # latency = sizeofbucket(limit) / rate (peakrate?)
        #
        #   32kbit
        # -------- = latency = 320ms
        #  100kbps
        #
        #  -but then-
        # burst ([token] buffer) the largest number of instantaneous
        # tokens available (i.e, bucket size).

        tbf_args = ""
        DEFLIMIT = 1518 * 1
        DEFBURST = 1518 * 2
        try:
            tc_rate = o_rate["rate"]
            tc_rate = convert_number(tc_rate)
            limit = convert_number(o_rate.get("limit", DEFLIMIT))
            burst = convert_number(o_rate.get("burst", DEFBURST))
        except (KeyError, TypeError):
            tc_rate = convert_number(o_rate)
            limit = convert_number(DEFLIMIT)
            burst = convert_number(DEFBURST)
        tbf_args += f" rate {tc_rate/1000}kbit"
        if delay:
            # give an extra 1/10 of buffer space to handle delay
            tbf_args += f" limit {limit} burst {burst}"
        else:
            tbf_args += f" limit {limit} burst {burst}"

        return netem_args, tbf_args

    def set_intf_constraints(self, ifname, **constraints):
        """Set interface outbound constraints.

        Set outbound constraints (jitter, delay, rate) for an interface. All arguments
        may also be passed as a string and will be converted to numerical format. All
        arguments are also optional. If not specified then that existing constraint will
        be cleared.

        Args:
            ifname: the name of the interface
            delay (int): number of microseconds.
            jitter (int): number of microseconds.
            jitter-correlation (float): Percent correlation to previous (default 10%).
            loss (float): Percent of loss.
            loss-correlation (float): Percent correlation to previous (default 25%).
            rate (int): bits per second, string allows for use of
                {KMGTKiMiGiTi} prefixes "i" means K == 1024 otherwise K == 1000.
        """
        nsifname = self.get_ns_ifname(ifname)
        netem_args, tbf_args = self.get_linux_tc_args(nsifname, constraints)
        count = 1
        selector = f"root handle {count}:"
        if netem_args:
            self.cmd_raises(
                f"tc qdisc add dev {nsifname} {selector} netem {netem_args}"
            )
            count += 1
            selector = f"parent {count-1}: handle {count}"
        # Place rate limit after delay otherwise limit/burst too complex
        if tbf_args:
            self.cmd_raises(f"tc qdisc add dev {nsifname} {selector} tbf {tbf_args}")

        self.cmd_raises(f"tc qdisc show dev {nsifname}")


class LinuxNamespace(Commander, InterfaceMixin):
    """A linux Namespace.

    An object that creates and executes commands in a linux namespace
    """

    def __init__(
        self,
        name,
        net=True,
        mount=True,
        uts=True,
        cgroup=False,
        ipc=False,
        pid=False,
        time=False,
        user=False,
        unshare_inline=False,
        set_hostname=True,
        private_mounts=None,
        **kwargs,
    ):
        """Create a new linux namespace.

        Args:
            name: Internal name for the namespace.
            net: Create network namespace.
            mount: Create network namespace.
            uts: Create UTS (hostname) namespace.
            cgroup: Create cgroup namespace.
            ipc: Create IPC namespace.
            pid: Create PID namespace, also mounts new /proc.
            time: Create time namespace.
            user: Create user namespace, also keeps capabilities.
            set_hostname: Set the hostname to `name`, uts must also be True.
            private_mounts: List of strings of the form
                "[/external/path:]/internal/path. If no external path is specified a
                tmpfs is mounted on the internal path. Any paths specified are first
                passed to `mkdir -p`.
            unshare_inline: Unshare the process itself rather than using a proxy.
            logger: Passed to superclass.
        """
        # logging.warning("LinuxNamespace: name %s kwargs %s", name, kwargs)

        super().__init__(name, **kwargs)

        unet = self.unet

        self.logger.debug("%s: creating", self)

        self.cwd = os.path.abspath(os.getcwd())

        self.nsflags = []
        self.ifnetns = {}
        self.uflags = 0
        self.p_ns_fds = None
        self.p_ns_fnames = None
        self.pid_ns = False
        self.init_pid = None
        self.unshare_inline = unshare_inline
        self.nsenter_fork = True

        #
        # Collect the namespaces to unshare
        #
        if hasattr(self, "proc_path") and self.proc_path:  # pylint: disable=no-member
            pp = Path(self.proc_path)  # pylint: disable=no-member
        else:
            pp = unet.proc_path if unet else Path("/proc")
        pp = pp.joinpath("%P%", "ns")

        flags = ""
        uflags = 0
        nslist = []
        nsflags = []
        if cgroup:
            nselm = "cgroup"
            nslist.append(nselm)
            nsflags.append(f"--{nselm}={pp / nselm}")
            flags += "C"
            uflags |= linux.CLONE_NEWCGROUP
        if ipc:
            nselm = "ipc"
            nslist.append(nselm)
            nsflags.append(f"--{nselm}={pp / nselm}")
            flags += "i"
            uflags |= linux.CLONE_NEWIPC
        if mount or pid:
            # We need a new mount namespace for pid
            nselm = "mnt"
            nslist.append(nselm)
            nsflags.append(f"--mount={pp / nselm}")
            mount = True
            flags += "m"
            uflags |= linux.CLONE_NEWNS
        if net:
            nselm = "net"
            nslist.append(nselm)
            nsflags.append(f"--{nselm}={pp / nselm}")
            # if pid:
            #     os.system(f"touch /tmp/netns-{name}")
            #     cmd.append(f"--net=/tmp/netns-{name}")
            # else:
            flags += "n"
            uflags |= linux.CLONE_NEWNET
        if pid:
            self.pid_ns = True
            # We look for this b/c the unshare pid will share with /sibn/init
            nselm = "pid_for_children"
            nslist.append(nselm)
            nsflags.append(f"--pid={pp / nselm}")
            flags += "p"
            uflags |= linux.CLONE_NEWPID
        if time:
            nselm = "time"
            # XXX time_for_children?
            nslist.append(nselm)
            nsflags.append(f"--{nselm}={pp / nselm}")
            flags += "T"
            uflags |= linux.CLONE_NEWTIME
        if user:
            nselm = "user"
            nslist.append(nselm)
            nsflags.append(f"--{nselm}={pp / nselm}")
            flags += "U"
            uflags |= linux.CLONE_NEWUSER
        if uts:
            nselm = "uts"
            nslist.append(nselm)
            nsflags.append(f"--{nselm}={pp / nselm}")
            flags += "u"
            uflags |= linux.CLONE_NEWUTS

        assert flags, "LinuxNamespace with no namespaces requested"

        # Should look path up using resources maybe...
        mutini_path = get_our_script_path("mutini")
        if not mutini_path:
            mutini_path = get_our_script_path("mutini.py")
        assert mutini_path
        cmd = [mutini_path, f"--unshare-flags={flags}", "-v"]
        fname = fsafe_name(self.name) + "-mutini.log"
        fname = (unet or self).rundir.joinpath(fname)
        stdout = open(fname, "w", encoding="utf-8")
        stderr = subprocess.STDOUT

        #
        # Save the current namespace info to compare against later
        #

        if not unet:
            nsdict = {x: os.readlink(f"/proc/self/ns/{x}") for x in nslist}
        else:
            nsdict = {
                x: os.readlink(f"{unet.proc_path}/{unet.pid}/ns/{x}") for x in nslist
            }

        #
        # (A) Basically we need to save the pid of the unshare call for nsenter.
        #
        # For `unet is not None` (node case) the level this exists at is based on wether
        # unet is using a forking nsenter or not. So if unet.nsenter_fork == True then
        # we need the child pid of the p.pid (child of pid returned to us), otherwise
        # unet.nsenter_fork == False and we just use p.pid as it will be unshare after
        # nsenter exec's it.
        #
        # For the `unet is None` (unet case) the unshare is at the top level or
        # non-existent so we always save the returned p.pid. If we are unshare_inline we
        # won't have a __pre_cmd but we can save our child_pid to kill later, otherwise
        # we set unet.pid to None b/c there's literally nothing to do.
        #
        # ---------------------------------------------------------------------------
        # Breakdown for nested (non-unet) namespace creation, and what PID
        # to use for __pre_cmd nsenter use.
        # ---------------------------------------------------------------------------
        #
        # tl;dr
        #   - for non-inline unshare: Use BBB with pid_for_children, unless none/none
        #     #then (AAA) returned
        #   - for inline unshare: use returned pid (AAA) with pid_for_children
        #
        # All commands use unet.popen to launch the unshare of mutini or cat.
        # mutini for PID unshare, otherwise cat. AAA is the returned pid BBB is the
        # child of the returned.
        #
        # Unshare Variant
        # ---------------
        #
        # Here we are running mutini if we are creating new pid namespace workspace,
        # cat otherwise.
        #
        # [PID+PID] pid tree looks like this:
        #
        # PID  NSPID PPID PGID
        # uuu    -   N/A  uuu  main unet process
        # AAA    -   uuu  AAA  nsenter (forking, from unet) (in unet namespaces -pid)
        # BBB    -   AAA  AAA  unshare --fork --kill-child (forking)
        # CCC    1   BBB  CCC  mutini (non-forking since it is pid 1 in new namespace)
        #
        # Use BBB if we use pid_for_children, CCC for all
        #
        # [PID+none] For non-pid workspace creation (but unet pid) we use cat and pid
        # tree looks like this:
        #
        # PID  PPID PGID
        # uuu  N/A  uuu  main unet process
        # AAA  uuu  AAA  nsenter (forking) (in unet namespaces -pid)
        # BBB  AAA  AAA  unshare -> cat (from unshare non-forking)
        #
        # Use BBB for all
        #
        # [none+PID] For pid workspace creation (but NOT unet pid) we use mutini and pid
        # tree looks like this:
        #
        # PID  NSPID PPID PGID
        # uuu    -   N/A  uuu  main unet process
        # AAA    -   uuu  AAA  nsenter -> unshare --fork --kill-child
        # BBB    1   AAA  AAA  mutini (non-forking since it is pid 1 in new namespace)
        #
        # Use AAA if we use pid_for_children, BBB for all
        #
        # [none+none] For non-pid workspace and non-pid unet we use cat and pid tree
        # looks like this:
        #
        # PID  PPID PGID
        # uuu  N/A  uuu  main unet process
        # AAA  uuu  AAA  nsenter -> unshare -> cat
        #
        # Use AAA for all, there's no BBB
        #
        # Inline-Unshare Variant
        # ----------------------
        #
        # For unshare_inline and new PID namespace we have unshared all but our PID
        # namespace, but our children end up in the new namespace so the fork popen
        # does is good enough.
        #
        # [PID+PID] pid tree looks like this:
        #
        # PID  NSPID PPID PGID
        # uuu    -   N/A  uuu  main unet process
        # AAA    -   uuu  AAA  unshare --fork --kill-child (forking)
        # BBB    1   AAA  BBB  mutini
        #
        # Use AAA if we use pid_for_children, BBB for all
        #
        # [PID+none] For non-pid workspace creation (but unet pid) we use cat and pid
        # tree looks like this:
        #
        # PID  PPID PGID
        # uuu  N/A  uuu  main unet process
        # AAA  uuu  AAA  unshare -> cat
        #
        # Use AAA for all
        #
        # [none+PID] For pid workspace creation (but NOT unet pid) we use mutini and pid
        # tree looks like this:
        #
        # PID  NSPID PPID PGID
        # uuu    -   N/A  uuu  main unet process
        # AAA    -   uuu  AAA  unshare --fork --kill-child
        # BBB    1   AAA  BBB  mutini
        #
        # Use AAA if we use pid_for_children, BBB for all
        #
        # [none+none] For non-pid workspace and non-pid unet we use cat and pid tree
        # looks like this:
        #
        # PID  PPID PGID
        # uuu  N/A  uuu  main unet process
        # AAA  uuu  AAA  unshare -> cat
        #
        # Use AAA for all.
        #
        #
        # ---------------------------------------------------------------------------
        # Breakdown for unet namespace creation, and what PID to use for __pre_cmd
        # ---------------------------------------------------------------------------
        #
        # tl;dr: save returned PID or nothing.
        #   - for non-inline unshare: Use AAA with pid_for_children (returned pid)
        #   - for inline unshare: no __precmd as the fork in popen is enough.
        #
        # Use commander to launch the unshare mutini/cat (for PID/none
        # workspace PID) for non-inline case. AAA is the returned pid BBB is the child
        # of the returned.
        #
        # Unshare Variant
        # ---------------
        #
        # Here we are running mutini if we are creating new pid namespace workspace,
        # cat otherwise.
        #
        # [PID] for unet pid creation pid tree looks like this:
        #
        # PID  NSPID PPID PGID
        # uuu    -   N/A  uuu  main unet process
        # AAA    -   uuu  AAA  unshare --fork --kill-child (forking)
        # BBB    1   AAA  BBB  mutini
        #
        # Use AAA if we use pid_for_children, BBB for all
        #
        # [none] for unet non-pid, pid tree looks like this:
        #
        # PID  PPID PGID
        # uuu  N/A  uuu  main unet process
        # AAA  uuu  AAA  unshare -> cat
        #
        # Use AAA for all
        #
        # Inline-Unshare Variant
        # -----------------------
        #
        # For unshare_inline and new PID namespace we have unshared all but our PID
        # namespace, but our children end up in the new namespace so the fork in popen
        # does is good enough.
        #
        # [PID] for unet pid creation pid tree looks like this:
        #
        # PID  NSPID PPID PGID
        # uuu    -   N/A  uuu  main unet process
        # AAA    1   uuu  AAA  mutini
        #
        # Save p / p.pid, but don't configure any nsenter, uneeded.
        #
        # Use nothing as the fork when doing a popen is enough to be in all the right
        # namepsaces.
        #
        # [none] for unet non-pid, pid tree looks like this:
        #
        # PID  PPID PGID
        # uuu  N/A  uuu  main unet process
        #
        # Nothing, no __pre_cmd.
        #
        #

        self.ppid = os.getppid()
        self.unshare_inline = unshare_inline
        if unshare_inline:
            assert unet is None
            self.uflags = uflags
            #
            # Open file descriptors for current namespaces for later restoration.
            #
            try:
                # pidfd_open is actually present in 5.4, is this 5.8 check for another
                # aspect of what the pidfd_open code is relying on, something in the
                # namespace code? If not we can simply check for os.pidfd_open() being
                # present as our compat module linux.py runtime patches it in if
                # supported by the kernel.
                kversion = [int(x) for x in platform.release().split("-")[0].split(".")]
                kvok = kversion[0] > 5 or (kversion[0] == 5 and kversion[1] >= 8)
            except ValueError:
                kvok = False
            if not kvok:
                # get list of namespace file descriptors before we unshare
                self.p_ns_fds = []
                self.p_ns_fnames = []
                tmpflags = uflags
                for i in range(0, 64):
                    v = 1 << i
                    if (tmpflags & v) == 0:
                        continue
                    tmpflags &= ~v
                    if v in linux.namespace_files:
                        path = os.path.join("/proc/self", linux.namespace_files[v])
                        if os.path.exists(path):
                            self.p_ns_fds.append(os.open(path, 0))
                            self.p_ns_fnames.append(f"{path} -> {os.readlink(path)}")
                            self.logger.debug(
                                "%s: saving old namespace fd %s (%s)",
                                self,
                                self.p_ns_fnames[-1],
                                self.p_ns_fds[-1],
                            )
                    if not tmpflags:
                        break
            else:
                self.p_ns_fds = None
                self.p_ns_fnames = None
                self.ppid_fd = linux.pidfd_open(self.ppid)

            self.logger.debug(
                "%s: unshare to new namespaces %s",
                self,
                linux.clone_flag_string(uflags),
            )

            linux.unshare(uflags)

            if not pid:
                p = None
                self.pid = None
                self.nsenter_fork = False
            else:
                # Need to fork to create the PID namespace, but we need to continue
                # running from the parent so that things like pytest work. We'll execute
                # a mutini process to manage the child init 1 duties.
                #
                # We (the parent pid) can no longer create threads, due to that being
                # restricted by the kernel. See EINVAL in clone(2).
                #
                p = commander.popen(
                    [mutini_path, "-v"],
                    stdin=subprocess.PIPE,
                    stdout=stdout,
                    stderr=stderr,
                    text=True,
                    # new session/pgid so signals don't propagate
                    start_new_session=True,
                    shell=False,
                )
                self.pid = p.pid
                self.nsenter_fork = False
        else:
            # Using cat and a stdin PIPE is nice as it will exit when we do. However,
            # we also detach it from the pgid so that signals do not propagate to it.
            # This is b/c it would exit early (e.g., ^C) then, at least the main munet
            # proc which has no other processes like frr daemons running, will take the
            # main network namespace with it, which will remove the bridges and the
            # veth pair (because the bridge side veth is deleted).
            self.logger.debug("%s: creating namespace process: %s", self, cmd)

            # Use the parent unet process if we have one this will cause us to inherit
            # the namespaces correctly even in the non-inline case.
            parent = self.unet if self.unet else commander

            p = parent.popen(
                cmd,
                stdin=subprocess.PIPE,
                stdout=stdout,
                stderr=stderr,
                text=True,
                shell=False,
                # start_new_session=not unet
                # preexec_fn=os.setsid if not unet else None,
                preexec_fn=os.setsid,
            )

            # The pid number returned is in the global pid namespace. For unshare_inline
            # this can be unfortunate b/c our /proc has been remounted in our new pid
            # namespace and won't contain global pid namespace pids. To solve for this
            # we get all the pid values for the process below.

            # See (A) above for when we need the child pid.
            self.logger.debug("%s: namespace process: %s", self, proc_str(p))
            self.pid = p.pid
            if unet and unet.nsenter_fork:
                assert not unet.unshare_inline
                # Need child pid of p.pid
                pgrep = unet.rootcmd.get_exec_path("pgrep")
                # a sing fork was done
                child_pid = unet.rootcmd.cmd_raises([pgrep, "-o", "-P", str(p.pid)])
                self.pid = int(child_pid.strip())
                self.logger.debug("%s: child of namespace process: %s", self, pid)

        self.p = p

        # Let's always have a valid value.
        if self.pid is None:
            self.pid = our_pid

        #
        # Let's find all our pids in the nested PID namespaces
        #
        if unet:
            proc_path = unet.proc_path
        else:
            proc_path = self.proc_path if hasattr(self, "proc_path") else "/proc"
        proc_path = f"{proc_path}/{self.pid}"

        pid_status = open(f"{proc_path}/status", "r", encoding="ascii").read()
        m = re.search(r"\nNSpid:((?:\t[0-9]+)+)\n", pid_status)
        self.pids = [int(x) for x in m.group(1).strip().split("\t")]
        assert self.pids[0] == self.pid

        self.logger.debug("%s: namespace scoped pids: %s", self, self.pids)

        # -----------------------------------------------
        # Now let's wait until unshare completes it's job
        # -----------------------------------------------
        timeout = Timeout(30)
        if self.pid is not None and self.pid != our_pid:
            while (not p or not p.poll()) and not timeout.is_expired():
                # check new namespace values against old (nsdict), unshare
                # can actually take a bit to complete.
                for fname in tuple(nslist):
                    # self.pid will be the global pid b/c we didn't unshare_inline
                    nspath = f"{proc_path}/ns/{fname}"
                    try:
                        nsf = os.readlink(nspath)
                    except OSError as error:
                        self.logger.debug(
                            "unswitched: error (ok) checking %s: %s", nspath, error
                        )
                        continue
                    if nsdict[fname] != nsf:
                        self.logger.debug(
                            "switched: original %s current %s", nsdict[fname], nsf
                        )
                        nslist.remove(fname)
                    elif unshare_inline:
                        logging.warning(
                            "unshare_inline not unshared %s == %s", nsdict[fname], nsf
                        )
                    else:
                        self.logger.debug(
                            "unswitched: current %s elapsed: %s", nsf, timeout.elapsed()
                        )
                if not nslist:
                    self.logger.debug(
                        "all done waiting for unshare after %s", timeout.elapsed()
                    )
                    break

                elapsed = int(timeout.elapsed())
                if elapsed <= 3:
                    time_mod.sleep(0.1)
                else:
                    self.logger.info(
                        "%s: unshare taking more than %ss: %s", self, elapsed, nslist
                    )
                    time_mod.sleep(1)

        if p is not None and p.poll():
            self.logger.error("%s: namespace process failed: %s", self, comm_error(p))
            assert p.poll() is None, "unshare failed"

        #
        # Setup the pre-command to enter the target namespace from the running munet
        # process using self.pid
        #

        if pid:
            nsenter_fork = True
        elif unet and unet.nsenter_fork:
            # if unet created a pid namespace we need to enter it since we aren't
            # entering a child pid namespace we created for the node. Otherwise
            # we have a /proc remounted under unet, but our process is running in
            # the root pid namepsace
            nselm = "pid_for_children"
            nsflags.append(f"--pid={pp / nselm}")
            nsenter_fork = True
        else:
            # We dont need a fork.
            nsflags.append("-F")
            nsenter_fork = False

        # Save nsenter values if running from root namespace
        # we need this for the unshare_inline case when run externally (e.g., from
        # within tmux server).
        root_nsflags = [x.replace("%P%", str(self.pid)) for x in nsflags]
        self.__root_base_pre_cmd = ["/usr/bin/nsenter", *root_nsflags]
        self.__root_pre_cmd = list(self.__root_base_pre_cmd)

        if unshare_inline:
            assert unet is None
            # We have nothing to do here since our process is now in the correct
            # namespaces and children will inherit from us, even the PID namespace will
            # be corrent b/c commands are run by first forking.
            self.nsenter_fork = False
            self.nsflags = []
            self.__base_pre_cmd = []
        else:
            # We will use nsenter
            self.nsenter_fork = nsenter_fork
            self.nsflags = nsflags
            self.__base_pre_cmd = list(self.__root_base_pre_cmd)

        self.__pre_cmd = list(self.__base_pre_cmd)

        # Always mark new mount namespaces as recursive private
        if mount:
            # if self.p is None and not pid:
            self.cmd_raises_nsonly("mount --make-rprivate /")

        # We need to remount the procfs for the new PID namespace, since we aren't using
        # unshare(1) which does that for us.
        if pid and unshare_inline:
            assert mount
            self.cmd_raises_nsonly("mount -t proc proc /proc")

        # We do not want cmd_status in child classes (e.g., container) for
        # the remaining setup calls in this __init__ function.

        if net:
            # Remount /sys to pickup any changes in the network, but keep root
            # /sys/fs/cgroup. This pattern could be made generic and supported for any
            # overlapping mounts
            if mount:
                tmpmnt = f"/tmp/cgm-{self.pid}"
                self.cmd_status_nsonly(
                    f"mkdir {tmpmnt} && mount --rbind /sys/fs/cgroup {tmpmnt}"
                )
                rc = o = e = None
                for i in range(0, 10):
                    rc, o, e = self.cmd_status_nsonly(
                        "mount -t sysfs sysfs /sys", warn=False
                    )
                    if not rc:
                        break
                    self.logger.debug(
                        "got error mounting new sysfs will retry: %s",
                        cmd_error(rc, o, e),
                    )
                    time_mod.sleep(1)
                else:
                    raise Exception(cmd_error(rc, o, e))

                self.cmd_status_nsonly(
                    f"mount --move {tmpmnt} /sys/fs/cgroup && rmdir {tmpmnt}"
                )

            # Original micronet code
            # self.cmd_raises_nsonly("mount -t sysfs sysfs /sys")
            # self.cmd_raises_nsonly(
            #     "mount -o rw,nosuid,nodev,noexec,relatime "
            #     "-t cgroup2 cgroup /sys/fs/cgroup"
            # )

        # Set the hostname to the namespace name
        if uts and set_hostname:
            self.cmd_status_nsonly("hostname " + self.name)
            nroot = subprocess.check_output("hostname")
            if unshare_inline or (unet and unet.unshare_inline):
                assert (
                    root_hostname != nroot
                ), f'hostname unchanged from "{nroot}" wanted "{self.name}"'
            else:
                # Assert that we didn't just change the host hostname
                assert (
                    root_hostname == nroot
                ), f'root hostname "{root_hostname}" changed to "{nroot}"!'

        if private_mounts:
            if isinstance(private_mounts, str):
                private_mounts = [private_mounts]
            for m in private_mounts:
                s = m.split(":", 1)
                if len(s) == 1:
                    self.tmpfs_mount(s[0])
                else:
                    self.bind_mount(s[0], s[1])

        # this will fail if running inside the namespace with PID
        if pid:
            o = self.cmd_nostatus_nsonly("ls -l /proc/1/ns")
        else:
            o = self.cmd_nostatus_nsonly("ls -l /proc/self/ns")

        self.logger.debug("namespaces:\n %s", o)

        # will cache the path, which is important in delete to avoid running a shell
        # which can hang during cleanup
        self.ip_path = get_exec_path_host("ip")
        if net:
            self.cmd_status_nsonly([self.ip_path, "link", "set", "lo", "up"])

        self.logger.info("%s: created", self)

    def _get_pre_cmd(self, use_str, use_pty, ns_only=False, root_level=False, **kwargs):
        """Get the pre-user-command values.

        The values returned here should be what is required to cause the user's command
        to execute in the correct context (e.g., namespace, container, sshremote).
        """
        del kwargs
        del ns_only
        del use_pty
        pre_cmd = self.__root_pre_cmd if root_level else self.__pre_cmd
        return shlex.join(pre_cmd) if use_str else list(pre_cmd)

    def tmpfs_mount(self, inner):
        self.logger.debug("Mounting tmpfs on %s", inner)
        self.cmd_raises("mkdir -p " + inner)
        self.cmd_raises("mount -n -t tmpfs tmpfs " + inner)

    def bind_mount(self, outer, inner):
        self.logger.debug("Bind mounting %s on %s", outer, inner)
        if commander.test("-f", outer):
            self.cmd_raises(f"mkdir -p {os.path.dirname(inner)} && touch {inner}")
        else:
            if not commander.test("-e", outer):
                commander.cmd_raises_nsonly(f"mkdir -p {outer}")
            self.cmd_raises(f"mkdir -p {inner}")
        self.cmd_raises("mount --rbind {} {} ".format(outer, inner))

    def add_netns(self, ns):
        self.logger.debug("Adding network namespace %s", ns)

        if os.path.exists("/run/netns/{}".format(ns)):
            self.logger.warning("%s: Removing existing nsspace %s", self, ns)
            try:
                self.delete_netns(ns)
            except Exception as ex:
                self.logger.warning(
                    "%s: Couldn't remove existing nsspace %s: %s",
                    self,
                    ns,
                    str(ex),
                    exc_info=True,
                )
        self.cmd_raises_nsonly([self.ip_path, "netns", "add", ns])

    def delete_netns(self, ns):
        self.logger.debug("Deleting network namespace %s", ns)
        self.cmd_raises_nsonly([self.ip_path, "netns", "delete", ns])

    def set_intf_netns(self, intf, ns, up=False):
        # In case a user hard-codes 1 thinking it "resets"
        ns = str(ns)
        if ns == "1":
            ns = str(self.pid)

        self.logger.debug("Moving interface %s to namespace %s", intf, ns)

        cmd = [self.ip_path, "link", "set", intf, "netns", ns]
        if up:
            cmd.append("up")
        self.intf_ip_cmd(intf, cmd)
        if ns == str(self.pid):
            # If we are returning then remove from dict
            if intf in self.ifnetns:
                del self.ifnetns[intf]
        else:
            self.ifnetns[intf] = ns

    def reset_intf_netns(self, intf):
        self.logger.debug("Moving interface %s to default namespace", intf)
        self.set_intf_netns(intf, str(self.pid))

    def intf_ip_cmd(self, intf, cmd):
        """Run an ip command, considering an interface's possible namespace."""
        if intf in self.ifnetns:
            if isinstance(cmd, list):
                assert cmd[0].endswith("ip")
                cmd[1:1] = ["-n", self.ifnetns[intf]]
            else:
                assert cmd.startswith("ip ")
                cmd = "ip -n " + self.ifnetns[intf] + cmd[2:]
        self.cmd_raises_nsonly(cmd)

    def intf_tc_cmd(self, intf, cmd):
        """Run a tc command, considering an interface's possible namespace."""
        if intf in self.ifnetns:
            if isinstance(cmd, list):
                assert cmd[0].endswith("tc")
                cmd[1:1] = ["-n", self.ifnetns[intf]]
            else:
                assert cmd.startswith("tc ")
                cmd = "tc -n " + self.ifnetns[intf] + cmd[2:]
        self.cmd_raises_nsonly(cmd)

    def set_ns_cwd(self, cwd: Union[str, Path]):
        """Common code for changing pre_cmd and pre_nscmd."""
        self.logger.debug("%s: new CWD %s", self, cwd)
        self.__root_pre_cmd = self.__root_base_pre_cmd + ["--wd=" + str(cwd)]
        if self.__pre_cmd:
            self.__pre_cmd = self.__base_pre_cmd + ["--wd=" + str(cwd)]
        elif self.unshare_inline:
            os.chdir(cwd)

    async def _async_delete(self):
        if type(self) == LinuxNamespace:  # pylint: disable=C0123
            self.logger.info("%s: deleting", self)
        else:
            self.logger.debug("%s: LinuxNamespace sub-class deleting", self)

        # Signal pid namespace proc to exit
        if (
            (self.p is None or self.p.pid != self.pid)
            and self.pid
            and self.pid != our_pid
        ):
            self.logger.debug(
                "cleanup separate pid %s from namespace proc pid %s",
                self.pid,
                self.p.pid if self.p else None,
            )
            await self.cleanup_pid(self.pid)

        if self.p is not None:
            self.logger.debug("cleanup namespace proc pid %s", self.p.pid)
            await self.async_cleanup_proc(self.p)

        # return to the previous namespace, need to do this in case anothe munet
        # is being created, especially when it plans to inherit the parent's (host)
        # namespace.
        if self.uflags:
            logging.info("restoring from inline unshare: cwd: %s", os.getcwd())
            # This only works in linux>=5.8
            if self.p_ns_fds is None:
                self.logger.debug(
                    "%s: restoring namespaces %s",
                    self,
                    linux.clone_flag_string(self.uflags),
                )
                # fd = linux.pidfd_open(self.ppid)
                fd = self.ppid_fd
                retry = 3
                for i in range(0, retry):
                    try:
                        linux.setns(fd, self.uflags)
                    except OSError as error:
                        self.logger.warning(
                            "%s: could not reset to old namespace fd %s: %s",
                            self,
                            fd,
                            error,
                        )
                        if i == retry - 1:
                            raise
                        time_mod.sleep(1)
                os.close(fd)
            else:
                while self.p_ns_fds:
                    fd = self.p_ns_fds.pop()
                    fname = self.p_ns_fnames.pop()
                    self.logger.debug(
                        "%s: restoring namespace from fd %s (%s)", self, fname, fd
                    )
                    retry = 3
                    for i in range(0, retry):
                        try:
                            linux.setns(fd, 0)
                            break
                        except OSError as error:
                            self.logger.warning(
                                "%s: could not reset to old namespace fd %s (%s): %s",
                                self,
                                fname,
                                fd,
                                error,
                            )
                            if i == retry - 1:
                                raise
                        time_mod.sleep(1)
                    os.close(fd)
                self.p_ns_fds = None
                self.p_ns_fnames = None
            logging.info("restored from unshare: cwd: %s", os.getcwd())

        self.__root_base_pre_cmd = ["/bin/false"]
        self.__base_pre_cmd = ["/bin/false"]
        self.__root_pre_cmd = ["/bin/false"]
        self.__pre_cmd = ["/bin/false"]

        await super()._async_delete()


class SharedNamespace(Commander):
    """Share another namespace.

    An object that executes commands in an existing pid's linux namespace
    """

    def __init__(self, name, pid=None, nsflags=None, **kwargs):
        """Share a linux namespace.

        Args:
            name: Internal name for the namespace.
            pid: PID of the process to share with.
            nsflags: nsenter flags to pass to inherit namespaces from
        """
        super().__init__(name, **kwargs)

        self.logger.debug("%s: Creating", self)

        self.cwd = os.path.abspath(os.getcwd())
        self.pid = pid if pid is not None else our_pid

        nsflags = (x.replace("%P%", str(self.pid)) for x in nsflags) if nsflags else []
        self.__base_pre_cmd = ["/usr/bin/nsenter", *nsflags] if nsflags else []
        self.__pre_cmd = self.__base_pre_cmd
        self.ip_path = self.get_exec_path("ip")

    def _get_pre_cmd(self, use_str, use_pty, ns_only=False, root_level=False, **kwargs):
        """Get the pre-user-command values.

        The values returned here should be what is required to cause the user's command
        to execute in the correct context (e.g., namespace, container, sshremote).
        """
        del kwargs
        del ns_only
        del use_pty
        assert not root_level
        return shlex.join(self.__pre_cmd) if use_str else list(self.__pre_cmd)

    def set_ns_cwd(self, cwd: Union[str, Path]):
        """Common code for changing pre_cmd and pre_nscmd."""
        self.logger.debug("%s: new CWD %s", self, cwd)
        self.__pre_cmd = self.__base_pre_cmd + ["--wd=" + str(cwd)]


class Bridge(SharedNamespace, InterfaceMixin):
    """A linux bridge."""

    next_ord = 1

    @classmethod
    def _get_next_id(cls):
        # Do not use `cls` here b/c that makes the variable class specific
        n = Bridge.next_ord
        Bridge.next_ord = n + 1
        return n

    def __init__(self, name=None, mtu=None, unet=None, **kwargs):
        """Create a linux Bridge."""
        self.id = self._get_next_id()
        if not name:
            name = "br{}".format(self.id)

        unet_pid = our_pid if unet.pid is None else unet.pid

        super().__init__(name, pid=unet_pid, nsflags=unet.nsflags, unet=unet, **kwargs)

        self.set_intf_basename(self.name + "-e")

        self.mtu = mtu

        self.logger.debug("Bridge: Creating")

        # assert len(self.name) <= 16  # Make sure fits in IFNAMSIZE
        self.cmd_raises(f"ip link delete {name} || true")
        self.cmd_raises(f"ip link add {name} type bridge")
        if self.mtu:
            self.cmd_raises(f"ip link set {name} mtu {self.mtu}")
        self.cmd_raises(f"ip link set {name} up")

        self.logger.debug("%s: Created, Running", self)

    def get_ifname(self, netname):
        return self.net_intfs[netname] if netname in self.net_intfs else None

    async def _async_delete(self):
        """Stop the bridge (i.e., delete the linux resources)."""
        if type(self) == Bridge:  # pylint: disable=C0123
            self.logger.info("%s: deleting", self)
        else:
            self.logger.debug("%s: Bridge sub-class deleting", self)

        rc, o, e = await self.async_cmd_status(
            [self.ip_path, "link", "show", self.name],
            stdin=subprocess.DEVNULL,
            start_new_session=True,
            warn=False,
        )
        if not rc:
            rc, o, e = await self.async_cmd_status(
                [self.ip_path, "link", "delete", self.name],
                stdin=subprocess.DEVNULL,
                start_new_session=True,
                warn=False,
            )
        if rc:
            self.logger.error(
                "%s: error deleting bridge %s: %s",
                self,
                self.name,
                cmd_error(rc, o, e),
            )
        await super()._async_delete()


class BaseMunet(LinuxNamespace):
    """Munet."""

    def __init__(
        self,
        name="munet",
        isolated=True,
        pid=True,
        rundir=None,
        pytestconfig=None,
        **kwargs,
    ):
        """Create a Munet."""
        # logging.warning("BaseMunet: %s", name)

        self.hosts = {}
        self.switches = {}
        self.links = {}
        self.macs = {}
        self.rmacs = {}
        self.isolated = isolated

        self.cli_server = None
        self.cli_sockpath = None
        self.cli_histfile = None
        self.cli_in_window_cmds = {}
        self.cli_run_cmds = {}

        #
        # We need a directory for various files
        #
        if not rundir:
            rundir = "/tmp/munet"
        self.rundir = Path(rundir)

        #
        # Always having a global /proc is required to keep things from exploding
        # complexity with nested new pid namespaces..
        #
        if pid:
            self.proc_path = Path(tempfile.mkdtemp(suffix="-proc", prefix="mu-"))
            logging.debug("%s: mounting /proc on proc_path %s", name, self.proc_path)
            linux.mount("proc", str(self.proc_path), "proc")
        else:
            self.proc_path = Path("/proc")

        #
        # Now create a root level commander that works regardless of whether we inline
        # unshare or not. Save it in the global variable as well
        #

        if not self.isolated:
            self.rootcmd = commander
        elif not pid:
            nsflags = (
                f"--mount={self.proc_path / '1/ns/mnt'}",
                f"--net={self.proc_path / '1/ns/net'}",
                f"--uts={self.proc_path / '1/ns/uts'}",
                # f"--ipc={self.proc_path / '1/ns/ipc'}",
                # f"--time={self.proc_path / '1/ns/time'}",
                # f"--cgroup={self.proc_path / '1/ns/cgroup'}",
            )
            self.rootcmd = SharedNamespace("root", pid=1, nsflags=nsflags)
        else:
            # XXX user
            nsflags = (
                # XXX Backing up PID namespace just doesn't work.
                # f"--pid={self.proc_path / '1/ns/pid_for_children'}",
                f"--mount={self.proc_path / '1/ns/mnt'}",
                f"--net={self.proc_path / '1/ns/net'}",
                f"--uts={self.proc_path / '1/ns/uts'}",
                # f"--ipc={self.proc_path / '1/ns/ipc'}",
                # f"--time={self.proc_path / '1/ns/time'}",
                # f"--cgroup={self.proc_path / '1/ns/cgroup'}",
            )
            self.rootcmd = SharedNamespace("root", pid=1, nsflags=nsflags)
        global roothost  # pylint: disable=global-statement

        roothost = self.rootcmd

        self.cfgopt = munet_config.ConfigOptionsProxy(pytestconfig)

        # This allows us to cleanup any leftover running munet's
        if "MUNET_PID" in os.environ:
            if os.environ["MUNET_PID"] != str(our_pid):
                logging.error(
                    "Found env MUNET_PID != our pid %s, instead its %s, changing",
                    our_pid,
                    os.environ["MUNET_PID"],
                )
        os.environ["MUNET_PID"] = str(our_pid)

        super().__init__(
            name, mount=True, net=isolated, uts=isolated, pid=pid, unet=None, **kwargs
        )

        # this is for testing purposes do not use
        if not BaseMunet.g_unet:
            BaseMunet.g_unet = self

        self.logger.debug("%s: Creating", self)

    def __getitem__(self, key):
        if key in self.switches:
            return self.switches[key]
        return self.hosts[key]

    def add_host(self, name, cls=LinuxNamespace, **kwargs):
        """Add a host to munet."""
        self.logger.debug("%s: add_host %s(%s)", self, cls.__name__, name)

        self.hosts[name] = cls(name, unet=self, **kwargs)

        # Create a new mounted FS for tracking nested network namespaces creatd by the
        # user with `ip netns add`

        # XXX why is this failing with podman???
        # self.hosts[name].tmpfs_mount("/run/netns")

        return self.hosts[name]

    def add_link(self, node1, node2, if1, if2, mtu=None, **intf_constraints):
        """Add a link between switch and node or 2 nodes.

        If constraints are given they are applied to each endpoint. See
        `InterfaceMixin::set_intf_constraints()` for more info.
        """
        isp2p = False

        try:
            name1 = node1.name
        except AttributeError:
            if node1 in self.switches:
                node1 = self.switches[node1]
            else:
                node1 = self.hosts[node1]
            name1 = node1.name

        try:
            name2 = node2.name
        except AttributeError:
            if node2 in self.switches:
                node2 = self.switches[node2]
            else:
                node2 = self.hosts[node2]
            name2 = node2.name

        if name1 in self.switches:
            assert name2 in self.hosts
        elif name2 in self.switches:
            assert name1 in self.hosts
            name1, name2 = name2, name1
            if1, if2 = if2, if1
        else:
            # p2p link
            assert name1 in self.hosts
            assert name2 in self.hosts
            isp2p = True

        lname = "{}:{}-{}:{}".format(name1, if1, name2, if2)
        self.logger.debug("%s: add_link %s%s", self, lname, " p2p" if isp2p else "")
        self.links[lname] = (name1, if1, name2, if2)

        # And create the veth now.
        if isp2p:
            lhost, rhost = self.hosts[name1], self.hosts[name2]
            lifname = "i1{:x}".format(lhost.pid)

            # Done at root level
            nsif1 = lhost.get_ns_ifname(if1)
            nsif2 = rhost.get_ns_ifname(if2)

            # Use pids[-1] to get the unet scoped pid for hosts
            self.cmd_raises_nsonly(
                f"ip link add {lifname} type veth peer name {nsif2}"
                f" netns {rhost.pids[-1]}"
            )
            self.cmd_raises_nsonly(f"ip link set {lifname} netns {lhost.pids[-1]}")

            lhost.cmd_raises_nsonly("ip link set {} name {}".format(lifname, nsif1))
            if mtu:
                lhost.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif1, mtu))
            lhost.cmd_raises_nsonly("ip link set {} up".format(nsif1))
            lhost.register_interface(if1)

            if mtu:
                rhost.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif2, mtu))
            rhost.cmd_raises_nsonly("ip link set {} up".format(nsif2))
            rhost.register_interface(if2)
        else:
            switch = self.switches[name1]
            rhost = self.hosts[name2]

            nsif1 = switch.get_ns_ifname(if1)
            nsif2 = rhost.get_ns_ifname(if2)

            if mtu is None:
                mtu = switch.mtu

            if len(nsif1) > 16:
                self.logger.error('"%s" len %s > 16', nsif1, len(nsif1))
            elif len(nsif2) > 16:
                self.logger.error('"%s" len %s > 16', nsif2, len(nsif2))
            assert len(nsif1) < 16 and len(nsif2) < 16  # Make sure fits in IFNAMSIZE

            self.logger.debug("%s: Creating veth pair for link %s", self, lname)

            # Use pids[-1] to get the unet scoped pid for hosts
            # switch is already in our namespace so nothing to convert.
            self.cmd_raises_nsonly(
                f"ip link add {nsif1} type veth peer name {nsif2}"
                f" netns {rhost.pids[-1]}"
            )

            if mtu:
                # if switch.mtu:
                #     # the switch interface should match the switch config
                #     switch.cmd_raises_nsonly(
                #         "ip link set {} mtu {}".format(if1, switch.mtu)
                #     )
                switch.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif1, mtu))
                rhost.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif2, mtu))

            switch.register_interface(if1)
            rhost.register_interface(if2)
            rhost.register_network(switch.name, if2)

            switch.cmd_raises_nsonly(f"ip link set {nsif1} master {switch.name}")

            switch.cmd_raises_nsonly(f"ip link set {nsif1} up")
            rhost.cmd_raises_nsonly(f"ip link set {nsif2} up")

        # Cache the MAC values, and reverse mapping
        self.get_mac(name1, nsif1)
        self.get_mac(name2, nsif2)

        # Setup interface constraints if provided
        if intf_constraints:
            node1.set_intf_constraints(if1, **intf_constraints)
            node2.set_intf_constraints(if2, **intf_constraints)

    def add_switch(self, name, cls=Bridge, **kwargs):
        """Add a switch to munet."""
        self.logger.debug("%s: add_switch %s(%s)", self, cls.__name__, name)
        self.switches[name] = cls(name, unet=self, **kwargs)
        return self.switches[name]

    def get_mac(self, name, ifname):
        if name in self.hosts:
            dev = self.hosts[name]
        else:
            dev = self.switches[name]

        nsifname = self.get_ns_ifname(ifname)

        if (name, ifname) not in self.macs:
            _, output, _ = dev.cmd_status_nsonly("ip -o link show " + nsifname)
            m = re.match(".*link/(loopback|ether) ([0-9a-fA-F:]+) .*", output)
            mac = m.group(2)
            self.macs[(name, ifname)] = mac
            self.rmacs[mac] = (name, ifname)

        return self.macs[(name, ifname)]

    async def _delete_link(self, lname):
        rname, rif = self.links[lname][2:4]
        host = self.hosts[rname]
        nsrif = host.get_ns_ifname(rif)

        self.logger.debug("%s: Deleting veth pair for link %s", self, lname)
        rc, o, e = await host.async_cmd_status_nsonly(
            [self.ip_path, "link", "delete", nsrif],
            stdin=subprocess.DEVNULL,
            start_new_session=True,
            warn=False,
        )
        if rc:
            self.logger.error("Err del veth pair %s: %s", lname, cmd_error(rc, o, e))

    async def _delete_links(self):
        # for x in self.links:
        #     await self._delete_link(x)
        return await asyncio.gather(*[self._delete_link(x) for x in self.links])

    async def _async_delete(self):
        """Delete the munet topology."""
        # logger = self.logger if False else logging
        logger = self.logger
        if type(self) == BaseMunet:  # pylint: disable=C0123
            logger.info("%s: deleting.", self)
        else:
            logger.debug("%s: BaseMunet sub-class deleting.", self)

        logger.debug("Deleting links")
        try:
            await self._delete_links()
        except Exception as error:
            logger.error("%s: error deleting links: %s", self, error, exc_info=True)

        logger.debug("Deleting hosts and bridges")
        try:
            # Delete hosts and switches, wait for them all to complete
            # even if there is an exception.
            htask = [x.async_delete() for x in self.hosts.values()]
            stask = [x.async_delete() for x in self.switches.values()]
            await asyncio.gather(*htask, *stask, return_exceptions=True)
        except Exception as error:
            logger.error(
                "%s: error deleting hosts and switches: %s", self, error, exc_info=True
            )

        self.links = {}
        self.hosts = {}
        self.switches = {}

        try:
            if self.cli_server:
                self.cli_server.cancel()
                self.cli_server = None
            if self.cli_sockpath:
                await self.async_cmd_status(
                    "rm -rf " + os.path.dirname(self.cli_sockpath)
                )
                self.cli_sockpath = None
        except Exception as error:
            logger.error(
                "%s: error cli server or sockpaths: %s", self, error, exc_info=True
            )

        try:
            if self.cli_histfile:
                readline.write_history_file(self.cli_histfile)
                self.cli_histfile = None
        except Exception as error:
            logger.error(
                "%s: error saving history file: %s", self, error, exc_info=True
            )

        # XXX for some reason setns during the delete is changing our dir to /.
        cwd = os.getcwd()

        try:
            await super()._async_delete()
        except Exception as error:
            logger.error(
                "%s: error deleting parent classes: %s", self, error, exc_info=True
            )
        os.chdir(cwd)

        try:
            if self.proc_path and str(self.proc_path) != "/proc":
                logger.debug("%s: umount, remove proc_path %s", self, self.proc_path)
                linux.umount(str(self.proc_path), 0)
                os.rmdir(self.proc_path)
        except Exception as error:
            logger.warning(
                "%s: error umount and removing proc_path %s: %s",
                self,
                self.proc_path,
                error,
                exc_info=True,
            )
            try:
                linux.umount(str(self.proc_path), linux.MNT_DETACH)
            except Exception as error2:
                logger.error(
                    "%s: error umount with detach proc_path %s: %s",
                    self,
                    self.proc_path,
                    error2,
                    exc_info=True,
                )

        if BaseMunet.g_unet == self:
            BaseMunet.g_unet = None


BaseMunet.g_unet = None

if True:  # pylint: disable=using-constant-test

    class ShellWrapper:
        """A Read-Execute-Print-Loop (REPL) interface.

        A newline or prompt changing command should be sent to the
        spawned child prior to creation as the `prompt` will be `expect`ed
        """

        def __init__(
            self,
            spawn,
            prompt,
            continuation_prompt=None,
            extra_init_cmd=None,
            will_echo=False,
            escape_ansi=False,
        ):
            self.echo = will_echo
            self.escape = (
                re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]") if escape_ansi else None
            )

            logging.debug(
                'ShellWraper: prompt "%s" will_echo %s child.echo %s',
                prompt,
                will_echo,
                spawn.echo,
            )

            self.child = spawn
            if self.child.echo:
                logging.info("Setting child to echo")
                self.child.setecho(False)
                self.child.waitnoecho()
                assert not self.child.echo

            self.prompt = prompt
            self.cont_prompt = continuation_prompt

            # Use expect_exact if we can as it should be faster
            self.expects = [prompt]
            if re.escape(prompt) == prompt and hasattr(self.child, "expect_exact"):
                self._expectf = self.child.expect_exact
            else:
                self._expectf = self.child.expect
            if continuation_prompt:
                self.expects.append(continuation_prompt)
                if re.escape(continuation_prompt) != continuation_prompt:
                    self._expectf = self.child.expect

            if extra_init_cmd:
                if isinstance(extra_init_cmd, str):
                    extra_init_cmd = [extra_init_cmd]
                for ecmd in extra_init_cmd:
                    self.expect_prompt()
                    self.child.sendline(ecmd)
            self.expect_prompt()

        def expect_prompt(self, timeout=-1):
            return self._expectf(self.expects, timeout=timeout)

        def run_command(self, command, timeout=-1):
            """Pexpect REPLWrapper compatible run_command.

            This will split `command` into lines and feed each one to the shell.

            Args:
                command: string of commands separated by newlines, a trailing
                    newline will cause and empty line to be sent.
                timeout: pexpect timeout value.
            """
            lines = command.splitlines()
            if command[-1] == "\n":
                lines.append("")
            output = ""
            index = 0
            for line in lines:
                self.child.sendline(line)
                index = self.expect_prompt(timeout=timeout)
                output += self.child.before

            if index:
                if hasattr(self.child, "kill"):
                    self.child.kill(signal.SIGINT)
                else:
                    self.child.send("\x03")
                self.expect_prompt(timeout=30 if self.child.timeout is None else -1)
                raise ValueError("Continuation prompt found at end of commands")

            if self.escape:
                output = self.escape.sub("", output)

            return output

        def cmd_nostatus(self, cmd, timeout=-1):
            r"""Execute a shell command.

            Returns:
                (strip/cleaned \r) output
            """
            output = self.run_command(cmd, timeout)
            output = output.replace("\r\n", "\n")
            if self.echo:
                # remove the command
                idx = output.find(cmd)
                if idx == -1:
                    logging.warning(
                        "Didn't find command ('%s') in expected output ('%s')",
                        cmd,
                        output,
                    )
                else:
                    # Remove up to and including the command from the output stream
                    output = output[idx + len(cmd) :]

            return output.replace("\r", "").strip()

        def cmd_status(self, cmd, timeout=-1):
            r"""Execute a shell command.

            Returns:
                status and (strip/cleaned \r) output
            """
            # Run the command getting the output
            output = self.cmd_nostatus(cmd, timeout)

            # Now get the status
            scmd = "echo $?"
            rcstr = self.run_command(scmd)
            rcstr = rcstr.replace("\r\n", "\n")
            if self.echo:
                # remove the command
                idx = rcstr.find(scmd)
                if idx == -1:
                    if self.echo:
                        logging.warning(
                            "Didn't find status ('%s') in expected output ('%s')",
                            scmd,
                            rcstr,
                        )
                    try:
                        rc = int(rcstr)
                    except Exception:
                        rc = 255
                else:
                    rcstr = rcstr[idx + len(scmd) :].strip()
            try:
                rc = int(rcstr)
            except ValueError as error:
                logging.error(
                    "%s: error with expected status output: %s: %s",
                    self,
                    error,
                    rcstr,
                    exc_info=True,
                )
                rc = 255
            return rc, output

        def cmd_raises(self, cmd, timeout=-1):
            r"""Execute a shell command.

            Returns:
                (strip/cleaned \r) ouptut

            Raises:
               CalledProcessError: on non-zero exit status
            """
            rc, output = self.cmd_status(cmd, timeout)
            if rc:
                raise CalledProcessError(rc, cmd, output)
            return output


# ---------------------------
# Root level utility function
# ---------------------------


def get_exec_path(binary):
    return commander.get_exec_path(binary)


def get_exec_path_host(binary):
    return commander.get_exec_path(binary)


def get_our_script_path(script):
    # would be nice to find this w/o using a path lookup
    sdir = os.path.dirname(os.path.abspath(__file__))
    spath = os.path.join(sdir, script)
    if os.path.exists(spath):
        return spath
    return get_exec_path(script)


commander = Commander("munet")
roothost = None