Adding upstream version 0.8.upstream/0.8upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 2184 insertions, 0 deletions

diff --git a/nvmet.service b/nvmet.service
new file mode 100644
index 0000000..6f97a91
--- /dev/null
+++ b/nvmet.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=Restore NVMe kernel target configuration
+Requires=sys-kernel-config.mount
+After=sys-kernel-config.mount network-online.target local-fs.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/sbin/nvmetcli restore
+ExecStop=/usr/sbin/nvmetcli clear
+SyslogIdentifier=nvmetcli
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/nvmet/__init__.py b/nvmet/__init__.py
new file mode 100644
index 0000000..cf172bd
--- /dev/null
+++ b/nvmet/__init__.py
@@ -0,0 +1,2 @@
+from .nvme import Root, Subsystem, Namespace, Port, Host, Referral, ANAGroup,\
+    DEFAULT_SAVE_FILE
diff --git a/nvmet/nvme.py b/nvmet/nvme.py
new file mode 100644
index 0000000..59efdb5
--- /dev/null
+++ b/nvmet/nvme.py
@@ -0,0 +1,928 @@
+'''
+Implements access to the NVMe target configfs hierarchy
+
+Copyright (c) 2011-2013 by Datera, Inc.
+Copyright (c) 2011-2014 by Red Hat, Inc.
+Copyright (c) 2016 by HGST, a Western Digital Company.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+'''
+
+import os
+import stat
+import uuid
+import json
+from glob import iglob as glob
+from six import iteritems, moves
+
+DEFAULT_SAVE_FILE = '/etc/nvmet/config.json'
+
+
+class CFSError(Exception):
+    '''
+    Generic slib error.
+    '''
+    pass
+
+
+class CFSNotFound(CFSError):
+    '''
+    The underlying configfs object does not exist. Happens when
+    calling methods of an object that is instantiated but have
+    been deleted from configfs, or when trying to lookup an
+    object that does not exist.
+    '''
+    pass
+
+
+class CFSNode(object):
+
+    configfs_dir = '/sys/kernel/config/nvmet'
+
+    def __init__(self):
+        self._path = self.configfs_dir
+        self._enable = None
+        self.attr_groups = []
+
+    def __eq__(self, other):
+        return self._path == other._path
+
+    def __ne__(self, other):
+        return self._path != other._path
+
+    def _get_path(self):
+        return self._path
+
+    def _create_in_cfs(self, mode):
+        '''
+        Creates the configFS node if it does not already exist, depending on
+        the mode.
+        any -> makes sure it exists, also works if the node already does exist
+        lookup -> make sure it does NOT exist
+        create -> create the node which must not exist beforehand
+        '''
+        if mode not in ['any', 'lookup', 'create']:
+            raise CFSError("Invalid mode: %s" % mode)
+        if self.exists and mode == 'create':
+            raise CFSError("This %s already exists in configFS" %
+                           self.__class__.__name__)
+        elif not self.exists and mode == 'lookup':
+            raise CFSNotFound("No such %s in configfs: %s" %
+                              (self.__class__.__name__, self.path))
+
+        if not self.exists:
+            try:
+                os.mkdir(self.path)
+            except:
+                raise CFSError("Could not create %s in configFS" %
+                               self.__class__.__name__)
+        self.get_enable()
+
+    def _exists(self):
+        return os.path.isdir(self.path)
+
+    def _check_self(self):
+        if not self.exists:
+            raise CFSNotFound("This %s does not exist in configFS" %
+                              self.__class__.__name__)
+
+    def list_attrs(self, group, writable=None):
+        '''
+        @param group: The attribute group
+        @param writable: If None (default), returns all attributes, if True,
+        returns read-write attributes, if False, returns just the read-only
+        attributes.
+        @type writable: bool or None
+        @return: A list of existing attribute names as strings.
+        '''
+        self._check_self()
+
+        names = [os.path.basename(name).split('_', 1)[1]
+                 for name in glob("%s/%s_*" % (self._path, group))
+                     if os.path.isfile(name)]
+
+        if writable is True:
+            names = [name for name in names
+                     if self._attr_is_writable(group, name)]
+        elif writable is False:
+            names = [name for name in names
+                     if not self._attr_is_writable(group, name)]
+
+        names.sort()
+        return names
+
+    def _attr_is_writable(self, group, name):
+        s = os.stat("%s/%s_%s" % (self._path, group, name))
+        return s[stat.ST_MODE] & stat.S_IWUSR
+
+    def set_attr(self, group, attribute, value):
+        '''
+        Sets the value of a named attribute.
+        The attribute must exist in configFS.
+        @param group: The attribute group
+        @param attribute: The attribute's name.
+        @param value: The attribute's value.
+        @type value: string
+        '''
+        self._check_self()
+        path = "%s/%s_%s" % (self.path, str(group), str(attribute))
+
+        if not os.path.isfile(path):
+            raise CFSError("Cannot find attribute: %s" % path)
+
+        if self._enable:
+            raise CFSError("Cannot set attribute while %s is enabled" %
+                           self.__class__.__name__)
+
+        try:
+            with open(path, 'w') as file_fd:
+                file_fd.write(str(value))
+        except Exception as e:
+            raise CFSError("Cannot set attribute %s: %s" % (path, e))
+
+    def get_attr(self, group, attribute):
+        '''
+        Gets the value of a named attribute.
+        @param group: The attribute group
+        @param attribute: The attribute's name.
+        @return: The named attribute's value, as a string.
+        '''
+        self._check_self()
+        path = "%s/%s_%s" % (self.path, str(group), str(attribute))
+        if not os.path.isfile(path):
+            raise CFSError("Cannot find attribute: %s" % path)
+
+        with open(path, 'r') as file_fd:
+            return file_fd.read().strip()
+
+    def get_enable(self):
+        self._check_self()
+        path = "%s/enable" % self.path
+        if not os.path.isfile(path):
+            return None
+
+        with open(path, 'r') as file_fd:
+            self._enable = int(file_fd.read().strip())
+        return self._enable
+
+    def set_enable(self, value):
+        self._check_self()
+        path = "%s/enable" % self.path
+
+        if not os.path.isfile(path) or self._enable is None:
+            raise CFSError("Cannot enable %s" % self.path)
+
+        try:
+            with open(path, 'w') as file_fd:
+                file_fd.write(str(value))
+        except Exception as e:
+            raise CFSError("Cannot enable %s: %s (%s)" %
+                           (self.path, e, value))
+        self._enable = value
+
+    def delete(self):
+        '''
+        If the underlying configFS object does not exist, this method does
+        nothing. If the underlying configFS object exists, this method attempts
+        to delete it.
+        '''
+        if self.exists:
+            os.rmdir(self.path)
+
+    path = property(_get_path,
+                    doc="Get the configFS object path.")
+    exists = property(_exists,
+            doc="Is True as long as the underlying configFS object exists. "
+                      + "If the underlying configFS objects gets deleted "
+                      + "either by calling the delete() method, or by any "
+                      + "other means, it will be False.")
+
+    def dump(self):
+        d = {}
+        for group in self.attr_groups:
+            a = {}
+            for i in self.list_attrs(group, writable=True):
+                a[str(i)] = self.get_attr(group, i)
+            d[str(group)] = a
+        if self._enable is not None:
+            d['enable'] = self._enable
+        return d
+
+    def _setup_attrs(self, attr_dict, err_func):
+        for group in self.attr_groups:
+            for name, value in iteritems(attr_dict.get(group, {})):
+                try:
+                    self.set_attr(group, name, value)
+                except CFSError as e:
+                    err_func(str(e))
+        enable = attr_dict.get('enable')
+        if enable is not None:
+            self.set_enable(enable)
+
+
+class Root(CFSNode):
+    def __init__(self):
+        super(Root, self).__init__()
+
+        if not os.path.isdir(self.configfs_dir):
+            self._modprobe('nvmet')
+
+        if not os.path.isdir(self.configfs_dir):
+            raise CFSError("%s does not exist.  Giving up." %
+                           self.configfs_dir)
+
+        self._path = self.configfs_dir
+        self._create_in_cfs('lookup')
+
+    def _modprobe(self, modname):
+        try:
+            from kmodpy import kmod
+
+            try:
+                kmod.Kmod().modprobe(modname, quiet=True)
+            except kmod.KmodError:
+                pass
+        except ImportError:
+            # Try the ctypes library included with the libkmod itself.
+            try:
+                import kmod
+                kmod.Kmod().modprobe(modname)
+            except Exception as e:
+                pass
+
+    def _list_subsystems(self):
+        self._check_self()
+
+        for d in os.listdir("%s/subsystems/" % self._path):
+            yield Subsystem(d, 'lookup')
+
+    subsystems = property(_list_subsystems,
+                          doc="Get the list of Subsystems.")
+
+    def _list_ports(self):
+        self._check_self()
+
+        for d in os.listdir("%s/ports/" % self._path):
+            yield Port(d, 'lookup')
+
+    ports = property(_list_ports,
+                doc="Get the list of Ports.")
+
+    def _list_hosts(self):
+        self._check_self()
+
+        for h in os.listdir("%s/hosts/" % self._path):
+            yield Host(h, 'lookup')
+
+    hosts = property(_list_hosts,
+                     doc="Get the list of Hosts.")
+
+    def save_to_file(self, savefile=None):
+        '''
+        Write the configuration in json format to a file.
+        '''
+        if savefile:
+            savefile = os.path.expanduser(savefile)
+        else:
+            savefile = DEFAULT_SAVE_FILE
+
+        savefile_abspath = os.path.abspath(savefile)
+        savefile_dir = os.path.dirname(savefile_abspath)
+        if not os.path.exists(savefile_dir):
+            os.makedirs(savefile_dir)
+
+        with open(savefile + ".temp", "w+") as f:
+            os.fchmod(f.fileno(), stat.S_IRUSR | stat.S_IWUSR)
+            f.write(json.dumps(self.dump(), sort_keys=True, indent=2))
+            f.write("\n")
+            f.flush()
+            os.fsync(f.fileno())
+
+        os.rename(savefile + ".temp", savefile)
+
+        # Sync the containing directory too
+        dir_fd = None
+        try:
+            dir_fd = os.open(savefile_dir, os.O_RDONLY)
+            os.fsync(dir_fd)
+        finally:
+            if dir_fd:
+                os.close(dir_fd)
+
+    def clear_existing(self):
+        '''
+        Remove entire current configuration.
+        '''
+
+        for p in self.ports:
+            p.delete()
+        for s in self.subsystems:
+            s.delete()
+        for h in self.hosts:
+            h.delete()
+
+    def restore(self, config, clear_existing=False, abort_on_error=False):
+        '''
+        Takes a dict generated by dump() and reconfigures the target to match.
+        Returns list of non-fatal errors that were encountered.
+        Will refuse to restore over an existing configuration unless
+        clear_existing is True.
+        '''
+        if clear_existing:
+            self.clear_existing()
+        else:
+            if any(self.subsystems):
+                raise CFSError("subsystems present, not restoring")
+
+        errors = []
+
+        if abort_on_error:
+            def err_func(err_str):
+                raise CFSError(err_str)
+        else:
+            def err_func(err_str):
+                errors.append(err_str + ", skipped")
+
+        # Create the hosts first because the subsystems reference them
+        for index, t in enumerate(config.get('hosts', [])):
+            if 'nqn' not in t:
+                err_func("'nqn' not defined in host %d" % index)
+                continue
+
+            Host.setup(t, err_func)
+
+        for index, t in enumerate(config.get('subsystems', [])):
+            if 'nqn' not in t:
+                err_func("'nqn' not defined in subsystem %d" % index)
+                continue
+
+            Subsystem.setup(t, err_func)
+
+        for index, t in enumerate(config.get('ports', [])):
+            if 'portid' not in t:
+                err_func("'portid' not defined in port %d" % index)
+                continue
+
+            Port.setup(self, t, err_func)
+
+        return errors
+
+    def restore_from_file(self, savefile=None, clear_existing=True,
+                          abort_on_error=False):
+        '''
+        Restore the configuration from a file in json format.
+        Returns a list of non-fatal errors. If abort_on_error is set,
+          it will raise the exception instead of continuing.
+        '''
+        if savefile:
+            savefile = os.path.expanduser(savefile)
+        else:
+            savefile = DEFAULT_SAVE_FILE
+
+        with open(savefile, "r") as f:
+            config = json.loads(f.read())
+            return self.restore(config, clear_existing=clear_existing,
+                                abort_on_error=abort_on_error)
+
+    def dump(self):
+        d = super(Root, self).dump()
+        d['subsystems'] = [s.dump() for s in self.subsystems]
+        d['ports'] = [p.dump() for p in self.ports]
+        d['hosts'] = [h.dump() for h in self.hosts]
+        return d
+
+
+class Subsystem(CFSNode):
+    '''
+    This is an interface to a NVMe Subsystem in configFS.
+    A Subsystem is identified by its NQN.
+    '''
+
+    def __repr__(self):
+        return "<Subsystem %s>" % self.nqn
+
+    def __init__(self, nqn=None, mode='any'):
+        '''
+        @param nqn: The Subsystems' NQN.
+            If no NQN is specified, one will be generated.
+        @type nqn: string
+        @param mode:An optional string containing the object creation mode:
+            - I{'any'} means the configFS object will be either looked up
+              or created.
+            - I{'lookup'} means the object MUST already exist configFS.
+            - I{'create'} means the object must NOT already exist in configFS.
+        @type mode:string
+        @return: A Subsystem object.
+        '''
+        super(Subsystem, self).__init__()
+
+        if nqn is None:
+            if mode == 'lookup':
+                raise CFSError("Need NQN for lookup")
+            nqn = self._generate_nqn()
+
+        self.nqn = nqn
+        self.attr_groups = ['attr']
+        self._path = "%s/subsystems/%s" % (self.configfs_dir, nqn)
+        self._create_in_cfs(mode)
+
+    def _generate_nqn(self):
+        prefix = "nqn.2014-08.org.nvmexpress:NVMf:uuid"
+        name = str(uuid.uuid4())
+        return "%s:%s" % (prefix, name)
+
+    def delete(self):
+        '''
+        Recursively deletes a Subsystem object.
+        This will delete all attached Namespace objects and then the
+        Subsystem itself.
+        '''
+        self._check_self()
+        for ns in self.namespaces:
+            ns.delete()
+        for h in self.allowed_hosts:
+            self.remove_allowed_host(h)
+        super(Subsystem, self).delete()
+
+    def _list_namespaces(self):
+        self._check_self()
+        for d in os.listdir("%s/namespaces/" % self._path):
+            yield Namespace(self, int(d), 'lookup')
+
+    namespaces = property(_list_namespaces,
+                          doc="Get the list of Namespaces for the Subsystem.")
+
+    def _list_allowed_hosts(self):
+        return [os.path.basename(name)
+                for name in os.listdir("%s/allowed_hosts/" % self._path)]
+
+    allowed_hosts = property(_list_allowed_hosts,
+                             doc="Get the list of Allowed Hosts for the Subsystem.")
+
+    def add_allowed_host(self, nqn):
+        '''
+        Enable access for the host identified by I{nqn} to the Subsystem
+        '''
+        try:
+            os.symlink("%s/hosts/%s" % (self.configfs_dir, nqn),
+                       "%s/allowed_hosts/%s" % (self._path, nqn))
+        except Exception as e:
+            raise CFSError("Could not symlink %s in configFS: %s" % (nqn, e))
+
+    def remove_allowed_host(self, nqn):
+        '''
+        Disable access for the host identified by I{nqn} to the Subsystem
+        '''
+        try:
+            os.unlink("%s/allowed_hosts/%s" % (self._path, nqn))
+        except Exception as e:
+            raise CFSError("Could not unlink %s in configFS: %s" % (nqn, e))
+
+    @classmethod
+    def setup(cls, t, err_func):
+        '''
+        Set up Subsystem objects based upon t dict, from saved config.
+        Guard against missing or bad dict items, but keep going.
+        Call 'err_func' for each error.
+        '''
+
+        if 'nqn' not in t:
+            err_func("'nqn' not defined for Subsystem")
+            return
+
+        try:
+            s = Subsystem(t['nqn'])
+        except CFSError as e:
+            err_func("Could not create Subsystem object: %s" % e)
+            return
+
+        for ns in t.get('namespaces', []):
+            Namespace.setup(s, ns, err_func)
+        for h in t.get('allowed_hosts', []):
+            s.add_allowed_host(h)
+
+        s._setup_attrs(t, err_func)
+
+    def dump(self):
+        d = super(Subsystem, self).dump()
+        d['nqn'] = self.nqn
+        d['namespaces'] = [ns.dump() for ns in self.namespaces]
+        d['allowed_hosts'] = self.allowed_hosts
+        return d
+
+
+class Namespace(CFSNode):
+    '''
+    This is an interface to a NVMe Namespace in configFS.
+    A Namespace is identified by its parent Subsystem and Namespace ID.
+    '''
+
+    MAX_NSID = 8192
+
+    def __repr__(self):
+        return "<Namespace %d>" % self.nsid
+
+    def __init__(self, subsystem, nsid=None, mode='any'):
+        '''
+        @param subsystem: The parent Subsystem object
+        @param nsid: The Namespace identifier
+            If no nsid is specified, the next free one will be used.
+        @type nsid: int
+        @param mode:An optional string containing the object creation mode:
+            - I{'any'} means the configFS object will be either looked up
+              or created.
+            - I{'lookup'} means the object MUST already exist configFS.
+            - I{'create'} means the object must NOT already exist in configFS.
+        @type mode:string
+        @return: A Namespace object.
+        '''
+        super(Namespace, self).__init__()
+
+        if not isinstance(subsystem, Subsystem):
+            raise CFSError("Invalid parent class")
+
+        if nsid is None:
+            if mode == 'lookup':
+                raise CFSError("Need NSID for lookup")
+
+            nsids = [n.nsid for n in subsystem.namespaces]
+            for index in moves.xrange(1, self.MAX_NSID + 1):
+                if index not in nsids:
+                    nsid = index
+                    break
+            if nsid is None:
+                raise CFSError("All NSIDs 1-%d in use" % self.MAX_NSID)
+        else:
+            nsid = int(nsid)
+            if nsid < 1 or nsid > self.MAX_NSID:
+                raise CFSError("NSID must be 1 to %d" % self.MAX_NSID)
+
+        self.attr_groups = ['device', 'ana']
+        self._subsystem = subsystem
+        self._nsid = nsid
+        self._path = "%s/namespaces/%d" % (self.subsystem.path, self.nsid)
+        self._create_in_cfs(mode)
+
+    def _get_subsystem(self):
+        return self._subsystem
+
+    def _get_nsid(self):
+        return self._nsid
+
+    def _get_grpid(self):
+        self._check_self()
+        _grpid = 0
+        path = "%s/ana_grpid" % self.path
+        if os.path.isfile(path):
+            with open(path, 'r') as file_fd:
+                _grpid = int(file_fd.read().strip())
+        return _grpid
+
+    def set_grpid(self, grpid):
+        self._check_self()
+        path = "%s/ana_grpid" % self.path
+        if os.path.isfile(path):
+            with open(path, 'w') as file_fd:
+                file_fd.write(str(grpid))
+
+    grpid = property(_get_grpid, doc="Get the ANA Group ID.")
+
+    subsystem = property(_get_subsystem,
+                         doc="Get the parent Subsystem object.")
+    nsid = property(_get_nsid, doc="Get the NSID as an int.")
+
+    @classmethod
+    def setup(cls, subsys, n, err_func):
+        '''
+        Set up a Namespace object based upon n dict, from saved config.
+        Guard against missing or bad dict items, but keep going.
+        Call 'err_func' for each error.
+        '''
+
+        if 'nsid' not in n:
+            err_func("'nsid' not defined for Namespace")
+            return
+
+        try:
+            ns = Namespace(subsys, n['nsid'])
+        except CFSError as e:
+            err_func("Could not create Namespace object: %s" % e)
+            return
+
+        ns._setup_attrs(n, err_func)
+        if 'ana_grpid' in n:
+            ns.set_grpid(int(n['ana_grpid']))
+
+    def dump(self):
+        d = super(Namespace, self).dump()
+        d['nsid'] = self.nsid
+        d['ana_grpid'] = self.grpid
+        return d
+
+
+class Port(CFSNode):
+    '''
+    This is an interface to a NVMe Port in configFS.
+    '''
+
+    MAX_PORTID = 8192
+
+    def __repr__(self):
+        return "<Port %d>" % self.portid
+
+    def __init__(self, portid, mode='any'):
+        super(Port, self).__init__()
+
+        self.attr_groups = ['addr', 'param']
+        self._portid = int(portid)
+        self._path = "%s/ports/%d" % (self.configfs_dir, self._portid)
+        self._create_in_cfs(mode)
+
+    def _get_portid(self):
+        return self._portid
+
+    portid = property(_get_portid, doc="Get the Port ID as an int.")
+
+    def _list_subsystems(self):
+        return [os.path.basename(name)
+                for name in os.listdir("%s/subsystems/" % self._path)]
+
+    subsystems = property(_list_subsystems,
+                          doc="Get the list of Subsystem for this Port.")
+
+    def add_subsystem(self, nqn):
+        '''
+        Enable access to the Subsystem identified by I{nqn} through this Port.
+        '''
+        try:
+            os.symlink("%s/subsystems/%s" % (self.configfs_dir, nqn),
+                       "%s/subsystems/%s" % (self._path, nqn))
+        except Exception as e:
+            raise CFSError("Could not symlink %s in configFS: %s" % (nqn, e))
+
+    def remove_subsystem(self, nqn):
+        '''
+        Disable access to the Subsystem identified by I{nqn} through this Port.
+        '''
+        try:
+            os.unlink("%s/subsystems/%s" % (self._path, nqn))
+        except Exception as e:
+            raise CFSError("Could not unlink %s in configFS: %s" % (nqn, e))
+
+    def delete(self):
+        '''
+        Recursively deletes a Port object.
+        '''
+        self._check_self()
+        for s in self.subsystems:
+            self.remove_subsystem(s)
+        for a in self.ana_groups:
+            a.delete()
+        for r in self.referrals:
+            r.delete()
+        super(Port, self).delete()
+
+    def _list_referrals(self):
+        self._check_self()
+        for d in os.listdir("%s/referrals/" % self._path):
+            yield Referral(self, d, 'lookup')
+
+    referrals = property(_list_referrals,
+                         doc="Get the list of Referrals for this Port.")
+
+    def _list_ana_groups(self):
+        self._check_self()
+        if os.path.isdir("%s/ana_groups/" % self._path):
+            for d in os.listdir("%s/ana_groups/" % self._path):
+                yield ANAGroup(self, int(d), 'lookup')
+
+    ana_groups = property(_list_ana_groups,
+                          doc="Get the list of ANA Groups for this Port.")
+
+    @classmethod
+    def setup(cls, root, n, err_func):
+        '''
+        Set up a Port object based upon n dict, from saved config.
+        Guard against missing or bad dict items, but keep going.
+        Call 'err_func' for each error.
+        '''
+
+        if 'portid' not in n:
+            err_func("'portid' not defined for Port")
+            return
+
+        try:
+            port = Port(n['portid'])
+        except CFSError as e:
+            err_func("Could not create Port object: %s" % e)
+            return
+
+        port._setup_attrs(n, err_func)
+        for s in n.get('subsystems', []):
+            port.add_subsystem(s)
+        for a in n.get('ana_groups', []):
+            ANAGroup.setup(port, a, err_func)
+        for r in n.get('referrals', []):
+            Referral.setup(port, r, err_func)
+
+    def dump(self):
+        d = super(Port, self).dump()
+        d['portid'] = self.portid
+        d['subsystems'] = self.subsystems
+        d['ana_groups'] = [a.dump() for a in self.ana_groups]
+        d['referrals'] = [r.dump() for r in self.referrals]
+        return d
+
+
+class Referral(CFSNode):
+    '''
+    This is an interface to a NVMe Referral in configFS.
+    '''
+
+    def __repr__(self):
+        return "<Referral %d>" % self.name
+
+    def __init__(self, port, name, mode='any'):
+        super(Referral, self).__init__()
+
+        if not isinstance(port, Port):
+            raise CFSError("Invalid parent class")
+
+        self.attr_groups = ['addr']
+        self.port = port
+        self._name = name
+        self._path = "%s/referrals/%s" % (self.port.path, self._name)
+        self._create_in_cfs(mode)
+
+    def _get_name(self):
+        return self._name
+
+    name = property(_get_name, doc="Get the Referral name.")
+
+    @classmethod
+    def setup(cls, port, n, err_func):
+        '''
+        Set up a Referral based upon n dict, from saved config.
+        Guard against missing or bad dict items, but keep going.
+        Call 'err_func' for each error.
+        '''
+
+        if 'name' not in n:
+            err_func("'name' not defined for Referral")
+            return
+
+        try:
+            r = Referral(port, n['name'])
+        except CFSError as e:
+            err_func("Could not create Referral object: %s" % e)
+            return
+
+        r._setup_attrs(n, err_func)
+
+    def dump(self):
+        d = super(Referral, self).dump()
+        d['name'] = self.name
+        return d
+
+
+class ANAGroup(CFSNode):
+    '''
+    This is an interface to a NVMe ANA Group in configFS.
+    '''
+
+    MAX_GRPID = 1024
+
+    def __repr__(self):
+        return "<ANA Group %d>" % self.grpid
+
+    def __init__(self, port, grpid, mode='any'):
+        super(ANAGroup, self).__init__()
+
+        if not os.path.isdir("%s/ana_groups" % port.path):
+            raise CFSError("ANA not supported")
+
+        if grpid is None:
+            if mode == 'lookup':
+                raise CFSError("Need grpid for lookup")
+
+            grpids = [n.grpid for n in port.ana_groups]
+            for index in moves.xrange(2, self.MAX_GRPID + 1):
+                if index not in grpids:
+                    grpid = index
+                    break
+            if grpid is None:
+                raise CFSError("All ANA Group IDs 1-%d in use" % self.MAX_GRPID)
+        else:
+            grpid = int(grpid)
+            if grpid < 1 or grpid > self.MAX_GRPID:
+                raise CFSError("GRPID %d must be 1 to %d" % (grpid, self.MAX_GRPID))
+
+        self.attr_groups = ['ana']
+        self._port = port
+        self._grpid = grpid
+        self._path = "%s/ana_groups/%d" % (self._port.path, self.grpid)
+        self._create_in_cfs(mode)
+
+    def _get_grpid(self):
+        return self._grpid
+
+    grpid = property(_get_grpid, doc="Get the ANA Group ID.")
+
+    @classmethod
+    def setup(cls, port, n, err_func):
+        '''
+        Set up an ANA Group object based upon n dict, from saved config.
+        Guard against missing or bad dict items, but keep going.
+        Call 'err_func' for each error.
+        '''
+
+        if 'grpid' not in n:
+            err_func("'grpid' not defined for ANA Group")
+            return
+
+        try:
+            a = ANAGroup(port, n['grpid'])
+        except CFSError as e:
+            err_func("Could not create ANA Group object: %s" % e)
+            return
+
+        a._setup_attrs(n, err_func)
+
+    def delete(self):
+        # ANA Group 1 is automatically created/deleted
+        if self.grpid != 1:
+            super(ANAGroup, self).delete()
+
+    def dump(self):
+        d = super(ANAGroup, self).dump()
+        d['grpid'] = self.grpid
+        return d
+
+
+class Host(CFSNode):
+    '''
+    This is an interface to a NVMe Host in configFS.
+    A Host is identified by its NQN.
+    '''
+
+    def __repr__(self):
+        return "<Host %s>" % self.nqn
+
+    def __init__(self, nqn, mode='any'):
+        '''
+        @param nqn: The Hosts's NQN.
+        @type nqn: string
+        @param mode:An optional string containing the object creation mode:
+            - I{'any'} means the configFS object will be either looked up
+              or created.
+            - I{'lookup'} means the object MUST already exist configFS.
+            - I{'create'} means the object must NOT already exist in configFS.
+        @type mode:string
+        @return: A Host object.
+        '''
+        super(Host, self).__init__()
+
+        self.nqn = nqn
+        self._path = "%s/hosts/%s" % (self.configfs_dir, nqn)
+        self._create_in_cfs(mode)
+
+    @classmethod
+    def setup(cls, t, err_func):
+        '''
+        Set up Host objects based upon t dict, from saved config.
+        Guard against missing or bad dict items, but keep going.
+        Call 'err_func' for each error.
+        '''
+
+        if 'nqn' not in t:
+            err_func("'nqn' not defined for Host")
+            return
+
+        try:
+            h = Host(t['nqn'])
+        except CFSError as e:
+            err_func("Could not create Host object: %s" % e)
+            return
+
+    def dump(self):
+        d = super(Host, self).dump()
+        d['nqn'] = self.nqn
+        return d
+
+
+def _test():
+    from doctest import testmod
+    testmod()
+
+if __name__ == "__main__":
+    _test()
diff --git a/nvmet/test_nvmet.py b/nvmet/test_nvmet.py
new file mode 100644
index 0000000..13e2f6c
--- /dev/null
+++ b/nvmet/test_nvmet.py
@@ -0,0 +1,484 @@
+
+import os
+import random
+import stat
+import string
+import unittest
+import nvmet.nvme as nvme
+
+# Default test devices are ram disks, but allow user to specify different
+# block devices or files.
+NVMET_TEST_DEVICES = os.getenv("NVMET_TEST_DEVICES",
+                               "/dev/ram0,/dev/ram1").split(',')
+
+
+def test_devices_present():
+    return len([x for x in NVMET_TEST_DEVICES
+                if os.path.exists(x) and
+                (stat.S_ISBLK(os.stat(x).st_mode) or os.path.isfile(x))]) >= 2
+
+
+class TestNvmet(unittest.TestCase):
+    def test_subsystem(self):
+        root = nvme.Root()
+        root.clear_existing()
+        for s in root.subsystems:
+            self.assertTrue(False, 'Found Subsystem after clear')
+
+        # create mode
+        s1 = nvme.Subsystem(nqn='testnqn1', mode='create')
+        self.assertIsNotNone(s1)
+        self.assertEqual(len(list(root.subsystems)), 1)
+
+        # any mode, should create
+        s2 = nvme.Subsystem(nqn='testnqn2', mode='any')
+        self.assertIsNotNone(s2)
+        self.assertEqual(len(list(root.subsystems)), 2)
+
+        # random name
+        s3 = nvme.Subsystem(mode='create')
+        self.assertIsNotNone(s3)
+        self.assertEqual(len(list(root.subsystems)), 3)
+
+        # duplicate
+        self.assertRaises(nvme.CFSError, nvme.Subsystem,
+                          nqn='testnqn1', mode='create')
+        self.assertEqual(len(list(root.subsystems)), 3)
+
+        # lookup using any, should not create
+        s = nvme.Subsystem(nqn='testnqn1', mode='any')
+        self.assertEqual(s1, s)
+        self.assertEqual(len(list(root.subsystems)), 3)
+
+        # lookup only
+        s = nvme.Subsystem(nqn='testnqn2', mode='lookup')
+        self.assertEqual(s2, s)
+        self.assertEqual(len(list(root.subsystems)), 3)
+
+        # lookup without nqn
+        self.assertRaises(nvme.CFSError, nvme.Subsystem, mode='lookup')
+
+        # and delete them all
+        for s in root.subsystems:
+            s.delete()
+        self.assertEqual(len(list(root.subsystems)), 0)
+
+    def test_namespace(self):
+        root = nvme.Root()
+        root.clear_existing()
+
+        s = nvme.Subsystem(nqn='testnqn', mode='create')
+        for n in s.namespaces:
+            self.assertTrue(False, 'Found Namespace in new Subsystem')
+
+        # create mode
+        n1 = nvme.Namespace(s, nsid=3, mode='create')
+        self.assertIsNotNone(n1)
+        self.assertEqual(len(list(s.namespaces)), 1)
+
+        # any mode, should create
+        n2 = nvme.Namespace(s, nsid=2, mode='any')
+        self.assertIsNotNone(n2)
+        self.assertEqual(len(list(s.namespaces)), 2)
+
+        # create without nsid, should pick lowest available
+        n3 = nvme.Namespace(s, mode='create')
+        self.assertIsNotNone(n3)
+        self.assertEqual(n3.nsid, 1)
+        self.assertEqual(len(list(s.namespaces)), 3)
+
+        n4 = nvme.Namespace(s, mode='create')
+        self.assertIsNotNone(n4)
+        self.assertEqual(n4.nsid, 4)
+        self.assertEqual(len(list(s.namespaces)), 4)
+
+        # duplicate
+        self.assertRaises(nvme.CFSError, nvme.Namespace, 1, mode='create')
+        self.assertEqual(len(list(s.namespaces)), 4)
+
+        # lookup using any, should not create
+        n = nvme.Namespace(s, nsid=3, mode='any')
+        self.assertEqual(n1, n)
+        self.assertEqual(len(list(s.namespaces)), 4)
+
+        # lookup only
+        n = nvme.Namespace(s, nsid=2, mode='lookup')
+        self.assertEqual(n2, n)
+        self.assertEqual(len(list(s.namespaces)), 4)
+
+        # lookup without nsid
+        self.assertRaises(nvme.CFSError, nvme.Namespace, None, mode='lookup')
+
+        # and delete them all
+        for n in s.namespaces:
+            n.delete()
+        self.assertEqual(len(list(s.namespaces)), 0)
+
+    @unittest.skipUnless(test_devices_present(),
+                         "Devices %s not available or suitable" % ','.join(NVMET_TEST_DEVICES))
+    def test_namespace_attrs(self):
+        root = nvme.Root()
+        root.clear_existing()
+
+        s = nvme.Subsystem(nqn='testnqn', mode='create')
+        n = nvme.Namespace(s, mode='create')
+
+        self.assertFalse(n.get_enable())
+        self.assertTrue('device' in n.attr_groups)
+        self.assertTrue('path' in n.list_attrs('device'))
+
+        # no device set yet, should fail
+        self.assertRaises(nvme.CFSError, n.set_enable, 1)
+
+        # now set a path and enable
+        n.set_attr('device', 'path', NVMET_TEST_DEVICES[0])
+        n.set_enable(1)
+        self.assertTrue(n.get_enable())
+
+        # test double enable
+        n.set_enable(1)
+
+        # test that we can't write to attrs while enabled
+        self.assertRaises(nvme.CFSError, n.set_attr, 'device', 'path',
+                          NVMET_TEST_DEVICES[1])
+        self.assertRaises(nvme.CFSError, n.set_attr, 'device', 'nguid',
+                          '15f7767b-50e7-4441-949c-75b99153dea7')
+
+        # disable: once and twice
+        n.set_enable(0)
+        n.set_enable(0)
+
+        # enable again, and remove while enabled
+        n.set_enable(1)
+        n.delete()
+
+    def test_recursive_delete(self):
+        root = nvme.Root()
+        root.clear_existing()
+
+        s = nvme.Subsystem(nqn='testnqn', mode='create')
+        n1 = nvme.Namespace(s, mode='create')
+        n2 = nvme.Namespace(s, mode='create')
+
+        s.delete()
+        self.assertEqual(len(list(root.subsystems)), 0)
+
+    def test_port(self):
+        root = nvme.Root()
+        root.clear_existing()
+        for p in root.ports:
+            self.assertTrue(False, 'Found Port after clear')
+
+        # create mode
+        p1 = nvme.Port(portid=0, mode='create')
+        self.assertIsNotNone(p1)
+        self.assertEqual(len(list(root.ports)), 1)
+
+        # any mode, should create
+        p2 = nvme.Port(portid=1, mode='any')
+        self.assertIsNotNone(p2)
+        self.assertEqual(len(list(root.ports)), 2)
+
+        # duplicate
+        self.assertRaises(nvme.CFSError, nvme.Port,
+                          portid=0, mode='create')
+        self.assertEqual(len(list(root.ports)), 2)
+
+        # lookup using any, should not create
+        p = nvme.Port(portid=0, mode='any')
+        self.assertEqual(p1, p)
+        self.assertEqual(len(list(root.ports)), 2)
+
+        # lookup only
+        p = nvme.Port(portid=1, mode='lookup')
+        self.assertEqual(p2, p)
+        self.assertEqual(len(list(root.ports)), 2)
+
+        # and delete them all
+        for p in root.ports:
+            p.delete()
+        self.assertEqual(len(list(root.ports)), 0)
+
+    def test_loop_port(self):
+        root = nvme.Root()
+        root.clear_existing()
+
+        s = nvme.Subsystem(nqn='testnqn', mode='create')
+        p = nvme.Port(portid=0, mode='create')
+
+        # subsystem doesn't exists, should fail
+        self.assertRaises(nvme.CFSError, p.add_subsystem, 'invalidnqn')
+
+        self.assertTrue('addr' in p.attr_groups)
+
+        # no trtype set yet, should fail
+        self.assertRaises(nvme.CFSError, p.add_subsystem, 'testnqn')
+
+        # now set trtype to loop and other attrs and enable
+        p.set_attr('addr', 'trtype', 'loop')
+        p.set_attr('addr', 'adrfam', 'ipv4')
+        p.set_attr('addr', 'traddr', '192.168.0.1')
+        p.set_attr('addr', 'treq', 'not required')
+        p.set_attr('addr', 'trsvcid', '1023')
+        p.add_subsystem('testnqn')
+
+        # test double add
+        self.assertRaises(nvme.CFSError, p.add_subsystem, 'testnqn')
+
+        # test that we can't write to attrs while enabled
+        self.assertRaises(nvme.CFSError, p.set_attr, 'addr', 'trtype',
+                          'rdma')
+        self.assertRaises(nvme.CFSError, p.set_attr, 'addr', 'adrfam',
+                          'ipv6')
+        self.assertRaises(nvme.CFSError, p.set_attr, 'addr', 'traddr',
+                          '10.0.0.1')
+        self.assertRaises(nvme.CFSError, p.set_attr, 'addr', 'treq',
+                          'required')
+        self.assertRaises(nvme.CFSError, p.set_attr, 'addr', 'trsvcid',
+                          '21')
+
+        # remove: once and twice
+        p.remove_subsystem('testnqn')
+        self.assertRaises(nvme.CFSError, p.remove_subsystem, 'testnqn')
+
+        # check that the attrs haven't been tampered with
+        self.assertEqual(p.get_attr('addr', 'trtype'), 'loop')
+        self.assertEqual(p.get_attr('addr', 'adrfam'), 'ipv4')
+        self.assertEqual(p.get_attr('addr', 'traddr'), '192.168.0.1')
+        self.assertEqual(p.get_attr('addr', 'treq'), 'not required')
+        self.assertEqual(p.get_attr('addr', 'trsvcid'), '1023')
+
+        # add again, and try to remove while enabled
+        p.add_subsystem('testnqn')
+        p.delete()
+
+    def test_host(self):
+        root = nvme.Root()
+        root.clear_existing()
+        for p in root.hosts:
+            self.assertTrue(False, 'Found Host after clear')
+
+        # create mode
+        h1 = nvme.Host(nqn='foo', mode='create')
+        self.assertIsNotNone(h1)
+        self.assertEqual(len(list(root.hosts)), 1)
+
+        # any mode, should create
+        h2 = nvme.Host(nqn='bar', mode='any')
+        self.assertIsNotNone(h2)
+        self.assertEqual(len(list(root.hosts)), 2)
+
+        # duplicate
+        self.assertRaises(nvme.CFSError, nvme.Host,
+                          'foo', mode='create')
+        self.assertEqual(len(list(root.hosts)), 2)
+
+        # lookup using any, should not create
+        h = nvme.Host('foo', mode='any')
+        self.assertEqual(h1, h)
+        self.assertEqual(len(list(root.hosts)), 2)
+
+        # lookup only
+        h = nvme.Host('bar', mode='lookup')
+        self.assertEqual(h2, h)
+        self.assertEqual(len(list(root.hosts)), 2)
+
+        # and delete them all
+        for h in root.hosts:
+            h.delete()
+        self.assertEqual(len(list(root.hosts)), 0)
+
+    def test_referral(self):
+        root = nvme.Root()
+        root.clear_existing()
+
+        # create port
+        p = nvme.Port(portid=1, mode='create')
+        self.assertEqual(len(list(p.referrals)), 0)
+
+        # create mode
+        r1 = nvme.Referral(p, name="1", mode='create')
+        self.assertIsNotNone(r1)
+        self.assertEqual(len(list(p.referrals)), 1)
+
+        # any mode, should create
+        r2 = nvme.Referral(p, name="2", mode='any')
+        self.assertIsNotNone(r2)
+        self.assertEqual(len(list(p.referrals)), 2)
+
+        # duplicate
+        self.assertRaises(nvme.CFSError, nvme.Referral,
+                          p, name="2", mode='create')
+        self.assertEqual(len(list(p.referrals)), 2)
+
+        # lookup using any, should not create
+        r = nvme.Referral(p, name="1", mode='any')
+        self.assertEqual(r1, r)
+        self.assertEqual(len(list(p.referrals)), 2)
+
+        # lookup only
+        r = nvme.Referral(p, name="2", mode='lookup')
+        self.assertEqual(r2, r)
+        self.assertEqual(len(list(p.referrals)), 2)
+
+        # non-existent lookup
+        self.assertRaises(nvme.CFSError, nvme.Referral, p, name="foo",
+                          mode='lookup')
+
+        # basic state
+        self.assertTrue('addr' in r.attr_groups)
+        self.assertFalse(r.get_enable())
+
+        # now set trtype to loop and other attrs and enable
+        r.set_attr('addr', 'trtype', 'loop')
+        r.set_attr('addr', 'adrfam', 'ipv4')
+        r.set_attr('addr', 'traddr', '192.168.0.1')
+        r.set_attr('addr', 'treq', 'not required')
+        r.set_attr('addr', 'trsvcid', '1023')
+        r.set_enable(1)
+
+        # test double enable
+        r.set_enable(1)
+
+        # test that we can't write to attrs while enabled
+        self.assertRaises(nvme.CFSError, r.set_attr, 'addr', 'trtype',
+                          'rdma')
+        self.assertRaises(nvme.CFSError, r.set_attr, 'addr', 'adrfam',
+                          'ipv6')
+        self.assertRaises(nvme.CFSError, r.set_attr, 'addr', 'traddr',
+                          '10.0.0.1')
+        self.assertRaises(nvme.CFSError, r.set_attr, 'addr', 'treq',
+                          'required')
+        self.assertRaises(nvme.CFSError, r.set_attr, 'addr', 'trsvcid',
+                          '21')
+
+        # disable: once and twice
+        r.set_enable(0)
+        r.set_enable(0)
+
+        # check that the attrs haven't been tampered with
+        self.assertEqual(r.get_attr('addr', 'trtype'), 'loop')
+        self.assertEqual(r.get_attr('addr', 'adrfam'), 'ipv4')
+        self.assertEqual(r.get_attr('addr', 'traddr'), '192.168.0.1')
+        self.assertEqual(r.get_attr('addr', 'treq'), 'not required')
+        self.assertEqual(r.get_attr('addr', 'trsvcid'), '1023')
+
+        # enable again, and try to remove while enabled
+        r.set_enable(1)
+        r.delete()
+
+        # remove the other one while disabled:
+        r1.delete()
+        self.assertEqual(len(list(p.referrals)), 0)
+
+    def test_allowed_hosts(self):
+        root = nvme.Root()
+
+        h = nvme.Host(nqn='hostnqn', mode='create')
+
+        s = nvme.Subsystem(nqn='testnqn', mode='create')
+
+        # add allowed_host
+        s.add_allowed_host(nqn='hostnqn')
+
+        # duplicate
+        self.assertRaises(nvme.CFSError, s.add_allowed_host, 'hostnqn')
+
+        # invalid
+        self.assertRaises(nvme.CFSError, s.add_allowed_host, 'invalid')
+
+        # remove again
+        s.remove_allowed_host('hostnqn')
+
+        # duplicate removal
+        self.assertRaises(nvme.CFSError, s.remove_allowed_host, 'hostnqn')
+
+        # invalid removal
+        self.assertRaises(nvme.CFSError, s.remove_allowed_host, 'foobar')
+
+    def test_invalid_input(self):
+        root = nvme.Root()
+        root.clear_existing()
+
+        self.assertRaises(nvme.CFSError, nvme.Subsystem,
+                          nqn='', mode='create')
+        self.assertRaises(nvme.CFSError, nvme.Subsystem,
+                          nqn='/', mode='create')
+
+        for l in [ 257, 512, 1024, 2048 ]:
+            toolong = ''.join(random.choice(string.ascii_lowercase)
+                              for i in range(l))
+            self.assertRaises(nvme.CFSError, nvme.Subsystem,
+                              nqn=toolong, mode='create')
+
+        discover_nqn = "nqn.2014-08.org.nvmexpress.discovery"
+        self.assertRaises(nvme.CFSError, nvme.Subsystem,
+                          nqn=discover_nqn, mode='create')
+
+        self.assertRaises(nvme.CFSError, nvme.Port,
+                          portid=1 << 17, mode='create')
+
+    @unittest.skipUnless(test_devices_present(),
+                         "Devices %s not available or suitable" % ','.join(
+                             NVMET_TEST_DEVICES))
+    def test_save_restore(self):
+        root = nvme.Root()
+        root.clear_existing()
+
+        h = nvme.Host(nqn='hostnqn', mode='create')
+
+        s = nvme.Subsystem(nqn='testnqn', mode='create')
+        s.add_allowed_host(nqn='hostnqn')
+
+        s2 = nvme.Subsystem(nqn='testnqn2', mode='create')
+        s2.set_attr('attr', 'allow_any_host', 1)
+
+        n = nvme.Namespace(s, nsid=42, mode='create')
+        n.set_attr('device', 'path', NVMET_TEST_DEVICES[0])
+        n.set_enable(1)
+
+        nguid = n.get_attr('device', 'nguid')
+
+        p = nvme.Port(portid=66, mode='create')
+        p.set_attr('addr', 'trtype', 'loop')
+        p.set_attr('addr', 'adrfam', 'ipv4')
+        p.set_attr('addr', 'traddr', '192.168.0.1')
+        p.set_attr('addr', 'treq', 'not required')
+        p.set_attr('addr', 'trsvcid', '1023')
+        p.add_subsystem('testnqn')
+
+        # save, clear, and restore
+        root.save_to_file('test.json')
+        root.clear_existing()
+        root.restore_from_file('test.json')
+
+        # additional restores should fai
+        self.assertRaises(nvme.CFSError, root.restore_from_file,
+                          'test.json', False)
+
+        # ... unless forced!
+        root.restore_from_file('test.json', True)
+
+        # rebuild our view of the world
+        h = nvme.Host(nqn='hostnqn', mode='lookup')
+        s = nvme.Subsystem(nqn='testnqn', mode='lookup')
+        s2 = nvme.Subsystem(nqn='testnqn2', mode='lookup')
+        n = nvme.Namespace(s, nsid=42, mode='lookup')
+        p = nvme.Port(portid=66, mode='lookup')
+
+        self.assertEqual(s.get_attr('attr', 'allow_any_host'), "0")
+        self.assertEqual(s2.get_attr('attr', 'allow_any_host'), "1")
+        self.assertIn('hostnqn', s.allowed_hosts)
+
+        # and check everything is still the same
+        self.assertTrue(n.get_enable())
+        self.assertEqual(n.get_attr('device', 'path'), NVMET_TEST_DEVICES[0])
+        self.assertEqual(n.get_attr('device', 'nguid'), nguid)
+
+        self.assertEqual(p.get_attr('addr', 'trtype'), 'loop')
+        self.assertEqual(p.get_attr('addr', 'adrfam'), 'ipv4')
+        self.assertEqual(p.get_attr('addr', 'traddr'), '192.168.0.1')
+        self.assertEqual(p.get_attr('addr', 'treq'), 'not required')
+        self.assertEqual(p.get_attr('addr', 'trsvcid'), '1023')
+        self.assertIn('testnqn', p.subsystems)
+        self.assertNotIn('testtnqn2', p.subsystems)
diff --git a/nvmetcli b/nvmetcli
new file mode 100755
index 0000000..d949891
--- /dev/null
+++ b/nvmetcli
@@ -0,0 +1,754 @@
+#!/usr/bin/python
+
+'''
+Frontend to access to the NVMe target configfs hierarchy
+
+Copyright (c) 2016 by HGST, a Western Digital Company.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+'''
+
+from __future__ import print_function
+
+import os
+import sys
+import configshell_fb as configshell
+import nvmet as nvme
+import errno
+from string import hexdigits
+import uuid
+
+
+def ngiud_set(nguid):
+    return any(c in hexdigits and c != '0' for c in nguid)
+
+
+class UINode(configshell.node.ConfigNode):
+    def __init__(self, name, parent=None, cfnode=None, shell=None):
+        configshell.node.ConfigNode.__init__(self, name, parent, shell)
+        self.cfnode = cfnode
+        if self.cfnode:
+            if self.cfnode.attr_groups:
+                for group in self.cfnode.attr_groups:
+                    self._init_group(group)
+        self.refresh()
+
+    def _init_group(self, group):
+        setattr(self.__class__, "ui_getgroup_%s" % group,
+                lambda self, attr:
+                    self.cfnode.get_attr(group, attr))
+        setattr(self.__class__, "ui_setgroup_%s" % group,
+                lambda self, attr, value:
+                    self.cfnode.set_attr(group, attr, value))
+
+        attrs = self.cfnode.list_attrs(group)
+        attrs_ro = self.cfnode.list_attrs(group, writable=False)
+        for attr in attrs:
+            writable = attr not in attrs_ro
+
+            name = "ui_desc_%s" % group
+            t, d = getattr(self.__class__, name, {}).get(attr, ('string', ''))
+            self.define_config_group_param(group, attr, t, d, writable)
+
+    def refresh(self):
+        self._children = set([])
+
+    def status(self):
+        return "None"
+
+    def ui_command_refresh(self):
+        '''
+        Refreshes and updates the objects tree from the current path.
+        '''
+        self.refresh()
+
+    def ui_command_status(self):
+        '''
+        Displays the current node's status summary.
+
+        SEE ALSO
+        ========
+        B{ls}
+        '''
+        self.shell.log.info("Status for %s: %s" % (self.path, self.status()))
+
+    def ui_command_saveconfig(self, savefile=None):
+        '''
+        Saves the current configuration to a file so that it can be restored
+        on next boot.
+        '''
+        node = self
+        while node.parent is not None:
+            node = node.parent
+        node.cfnode.save_to_file(savefile)
+
+
+class UIRootNode(UINode):
+    def __init__(self, shell):
+        UINode.__init__(self, '/', parent=None, cfnode=nvme.Root(),
+                        shell=shell)
+
+    def refresh(self):
+        self._children = set([])
+        UISubsystemsNode(self)
+        UIPortsNode(self)
+        UIHostsNode(self)
+
+    def ui_command_restoreconfig(self, savefile=None, clear_existing=False):
+        '''
+        Restores configuration from a file.
+        '''
+        errors = self.cfnode.restore_from_file(savefile, clear_existing)
+        self.refresh()
+
+        if errors:
+            raise configshell.ExecutionError(
+                "Configuration restored, %d errors:\n%s" %
+                (len(errors), "\n".join(errors)))
+
+
+class UISubsystemsNode(UINode):
+    def __init__(self, parent):
+        UINode.__init__(self, 'subsystems', parent)
+
+    def refresh(self):
+        self._children = set([])
+        for subsys in self.parent.cfnode.subsystems:
+            UISubsystemNode(self, subsys)
+
+    def ui_command_create(self, nqn=None):
+        '''
+        Creates a new target. If I{nqn} is omitted, then the new Subsystem
+        will be created using a randomly generated NQN.
+
+        SEE ALSO
+        ========
+        B{delete}
+        '''
+        subsystem = nvme.Subsystem(nqn, mode='create')
+        UISubsystemNode(self, subsystem)
+
+    def ui_command_delete(self, nqn):
+        '''
+        Recursively deletes the subsystem with the specified I{nqn}, and all
+        objects hanging under it.
+
+        SEE ALSO
+        ========
+        B{create}
+        '''
+        subsystem = nvme.Subsystem(nqn, mode='lookup')
+        subsystem.delete()
+        self.refresh()
+
+
+class UISubsystemNode(UINode):
+    ui_desc_attr = {
+        'allow_any_host': ('string', 'Allow access by any host if set to 1'),
+        'serial': ('string', 'Export serial number to hosts'),
+        'version': ('string', 'Export version number to hosts'),
+    }
+
+    def __init__(self, parent, cfnode):
+        UINode.__init__(self, cfnode.nqn, parent, cfnode)
+
+    def refresh(self):
+        self._children = set([])
+        UINamespacesNode(self)
+        UIAllowedHostsNode(self)
+
+    def summary(self):
+        info = []
+        info.append("version=" + self.cfnode.get_attr("attr", "version"))
+        info.append("allow_any=" +
+                    self.cfnode.get_attr("attr", "allow_any_host"))
+        info.append("serial=" + self.cfnode.get_attr("attr", "serial"))
+        return (", ".join(info), True)
+
+
+class UINamespacesNode(UINode):
+    def __init__(self, parent):
+        UINode.__init__(self, 'namespaces', parent)
+
+    def refresh(self):
+        self._children = set([])
+        for ns in self.parent.cfnode.namespaces:
+            UINamespaceNode(self, ns)
+
+    def ui_command_create(self, nsid=None):
+        '''
+        Creates a new namespace. If I{nsid} is omitted, then the next
+        available namespace id will be used.
+
+        SEE ALSO
+        ========
+        B{delete}
+        '''
+        namespace = nvme.Namespace(self.parent.cfnode, nsid, mode='create')
+        UINamespaceNode(self, namespace)
+
+    def ui_command_delete(self, nsid):
+        '''
+        Recursively deletes the namespace with the specified I{nsid}, and all
+        objects hanging under it.
+
+        SEE ALSO
+        ========
+        B{create}
+        '''
+        namespace = nvme.Namespace(self.parent.cfnode, nsid, mode='lookup')
+        namespace.delete()
+        self.refresh()
+
+
+class UINamespaceNode(UINode):
+    ui_desc_device = {
+        'path': ('string', 'Backing device path.'),
+        'nguid': ('string', 'Namspace Global Unique Identifier.'),
+        'uuid': ('string', 'Namespace Universally Unique Identifier.'),
+    }
+
+    def __init__(self, parent, cfnode):
+        UINode.__init__(self, str(cfnode.nsid), parent, cfnode)
+
+    def status(self):
+        if self.cfnode.get_enable():
+            return "enabled"
+        return "disabled"
+
+    def ui_command_enable(self):
+        '''
+        Enables the current Namespace.
+
+        SEE ALSO
+        ========
+        B{disable}
+        '''
+        if self.cfnode.get_enable():
+            self.shell.log.info("The Namespace is already enabled.")
+        else:
+            try:
+                self.cfnode.set_enable(1)
+                self.shell.log.info("The Namespace has been enabled.")
+            except Exception as e:
+                raise configshell.ExecutionError(
+                    "The Namespace could not be enabled.")
+
+    def ui_command_disable(self):
+        '''
+        Disables the current Namespace.
+
+        SEE ALSO
+        ========
+        B{enable}
+        '''
+        if not self.cfnode.get_enable():
+            self.shell.log.info("The Namespace is already disabled.")
+        else:
+            try:
+                self.cfnode.set_enable(0)
+                self.shell.log.info("The Namespace has been disabled.")
+            except Exception as e:
+                raise configshell.ExecutionError(
+                    "The Namespace could not be disabled.")
+
+    def ui_command_grpid(self, grpid):
+        '''
+        Sets the ANA Group ID of the current Namespace to I{grpid}
+        '''
+        try:
+            self.cfnode.set_grpid(grpid)
+        except Exception as e:
+            raise configshell.ExecutionError(
+                "Failed to set ANA Group ID for this Namespace.")
+
+    def summary(self):
+        info = []
+        info.append("path=" + self.cfnode.get_attr("device", "path"))
+        ns_uuid = self.cfnode.get_attr("device", "uuid")
+        if uuid.UUID(ns_uuid).int != 0:
+            info.append("uuid=" + str(ns_uuid))
+        ns_nguid = self.cfnode.get_attr("device", "nguid")
+        if ngiud_set(ns_nguid):
+            info.append("nguid=" + ns_nguid)
+        if self.cfnode.grpid != 0:
+            info.append("grpid=" + str(self.cfnode.grpid))
+        info.append("enabled" if self.cfnode.get_enable() else "disabled")
+        ns_enabled = self.cfnode.get_enable()
+        return (", ".join(info), True if ns_enabled == 1 else ns_enabled)
+
+
+class UIAllowedHostsNode(UINode):
+    def __init__(self, parent):
+        UINode.__init__(self, 'allowed_hosts', parent)
+
+    def refresh(self):
+        self._children = set([])
+        for host in self.parent.cfnode.allowed_hosts:
+            UIAllowedHostNode(self, host)
+
+    def ui_command_create(self, nqn):
+        '''
+        Grants access to parent subsystems to the host specified by I{nqn}.
+
+        SEE ALSO
+        ========
+        B{delete}
+        '''
+        self.parent.cfnode.add_allowed_host(nqn)
+        UIAllowedHostNode(self, nqn)
+
+    def ui_complete_create(self, parameters, text, current_param):
+        completions = []
+        if current_param == 'nqn':
+            for host in self.get_node('/hosts').children:
+                completions.append(host.cfnode.nqn)
+
+        if len(completions) == 1:
+            return [completions[0] + ' ']
+        else:
+            return completions
+
+    def ui_command_delete(self, nqn):
+        '''
+        Recursively deletes the namespace with the specified I{nsid}, and all
+        objects hanging under it.
+
+        SEE ALSO
+        ========
+        B{create}
+        '''
+        self.parent.cfnode.remove_allowed_host(nqn)
+        self.refresh()
+
+    def ui_complete_delete(self, parameters, text, current_param):
+        completions = []
+        if current_param == 'nqn':
+            for nqn in self.parent.cfnode.allowed_hosts:
+                completions.append(nqn)
+
+        if len(completions) == 1:
+            return [completions[0] + ' ']
+        else:
+            return completions
+
+
+class UIAllowedHostNode(UINode):
+    def __init__(self, parent, nqn):
+        UINode.__init__(self, nqn, parent)
+
+
+class UIPortsNode(UINode):
+    def __init__(self, parent):
+        UINode.__init__(self, 'ports', parent)
+
+    def refresh(self):
+        self._children = set([])
+        for port in self.parent.cfnode.ports:
+            UIPortNode(self, port)
+
+    def ui_command_create(self, portid=None):
+        '''
+        Creates a new NVMe port with portid I{portid}.
+
+        SEE ALSO
+        ========
+        B{delete}
+        '''
+        port = nvme.Port(portid, mode='create')
+        UIPortNode(self, port)
+
+    def ui_command_delete(self, portid):
+        '''
+        Recursively deletes the NVMe Port with the specified I{port}, and all
+        objects hanging under it.
+
+        SEE ALSO
+        ========
+        B{create}
+        '''
+        port = nvme.Port(portid, mode='lookup')
+        port.delete()
+        self.refresh()
+
+
+class UIPortNode(UINode):
+    ui_desc_addr = {
+        'adrfam': ('string', 'Address Family (e.g. ipv4 or fc)'),
+        'treq': ('string', 'Transport Security Requirements'),
+        'traddr': ('string',
+                   'Transport Address (e.g. IP Address or FC wwnn:wwpn)'),
+        'trsvcid': ('string', 'Transport Service ID (e.g. IP Port)'),
+        'trtype': ('string', 'Transport Type (e.g. rdma or loop or fc)'),
+    }
+    ui_desc_param = {
+        'inline_data_size': ('string', 'Port inline data size in bytes'),
+    }
+
+    def __init__(self, parent, cfnode):
+        UINode.__init__(self, str(cfnode.portid), parent, cfnode)
+        UIPortSubsystemsNode(self)
+        try:
+            next(cfnode.ana_groups)
+        except StopIteration:
+            pass
+        else:
+            UIANAGroupsNode(self)
+        UIReferralsNode(self)
+
+    def summary(self):
+        info = []
+        info.append("trtype=" + self.cfnode.get_attr("addr", "trtype"))
+        info.append("traddr=" + self.cfnode.get_attr("addr", "traddr"))
+        trsvcid = self.cfnode.get_attr("addr", "trsvcid")
+        if trsvcid != "none":
+            info.append("trsvcid=%s" % trsvcid)
+
+        '''
+        Support older target driver w/o the inline_data_size parameter
+        '''
+        try:
+            inline_data_size = self.cfnode.get_attr("param", "inline_data_size")
+        except Exception as e:
+            inline_data_size = "n/a"
+        if inline_data_size != "n/a":
+            info.append("inline_data_size=" + inline_data_size)
+        enabled = self.cfnode.subsystems or list(self.cfnode.referrals)
+        return (", ".join(info), True if enabled else 0)
+
+
+class UIPortSubsystemsNode(UINode):
+    def __init__(self, parent):
+        UINode.__init__(self, 'subsystems', parent)
+
+    def refresh(self):
+        self._children = set([])
+        for host in self.parent.cfnode.subsystems:
+            UIPortSubsystemNode(self, host)
+
+    def ui_command_create(self, nqn):
+        '''
+        Grants access to the subsystem specified by I{nqn} through the
+        parent port.
+
+        SEE ALSO
+        ========
+        B{delete}
+        '''
+        self.parent.cfnode.add_subsystem(nqn)
+        UIPortSubsystemNode(self, nqn)
+
+    def ui_complete_create(self, parameters, text, current_param):
+        completions = []
+        if current_param == 'nqn':
+            for subsys in self.get_node('/subsystems').children:
+                completions.append(subsys.cfnode.nqn)
+
+        if len(completions) == 1:
+            return [completions[0] + ' ']
+        else:
+            return completions
+
+    def ui_command_delete(self, nqn):
+        '''
+        Removes access to the subsystem specified by I{nqn} through the
+        parent port.
+
+        SEE ALSO
+        ========
+        B{create}
+        '''
+        self.parent.cfnode.remove_subsystem(nqn)
+        self.refresh()
+
+    def ui_complete_delete(self, parameters, text, current_param):
+        completions = []
+        if current_param == 'nqn':
+            for nqn in self.parent.cfnode.subsystems:
+                completions.append(nqn)
+
+        if len(completions) == 1:
+            return [completions[0] + ' ']
+        else:
+            return completions
+
+
+class UIPortSubsystemNode(UINode):
+    def __init__(self, parent, nqn):
+        UINode.__init__(self, nqn, parent)
+
+
+class UIReferralsNode(UINode):
+    def __init__(self, parent):
+        UINode.__init__(self, 'referrals', parent)
+
+    def refresh(self):
+        self._children = set([])
+        for r in self.parent.cfnode.referrals:
+            UIReferralNode(self, r)
+
+    def ui_command_create(self, name):
+        '''
+        Creates a new referral.
+
+        SEE ALSO
+        ========
+        B{delete}
+        '''
+        r = nvme.Referral(self.parent.cfnode, name, mode='create')
+        UIReferralNode(self, r)
+
+    def ui_command_delete(self, name):
+        '''
+        Deletes the referral with the specified I{name}.
+
+        SEE ALSO
+        ========
+        B{create}
+        '''
+        r = nvme.Referral(self.parent.cfnode, name, mode='lookup')
+        r.delete()
+        self.refresh()
+
+
+class UIReferralNode(UINode):
+    ui_desc_addr = {
+        'adrfam': ('string', 'Address Family (e.g. ipv4 or fc)'),
+        'treq': ('string', 'Transport Security Requirements'),
+        'traddr': ('string',
+                   'Transport Address (e.g. IP Address or FC wwnn:wwpn)'),
+        'trsvcid': ('string', 'Transport Service ID (e.g. IP Port)'),
+        'trtype': ('string', 'Transport Type (e.g. rdma or loop or fc)'),
+        'portid': ('number', 'Port identifier'),
+    }
+
+    def __init__(self, parent, cfnode):
+        UINode.__init__(self, cfnode.name, parent, cfnode)
+
+    def status(self):
+        if self.cfnode.get_enable():
+            return "enabled"
+        return "disabled"
+
+    def ui_command_enable(self):
+        '''
+        Enables the current Referral.
+
+        SEE ALSO
+        ========
+        B{disable}
+        '''
+        if self.cfnode.get_enable():
+            self.shell.log.info("The Referral is already enabled.")
+        else:
+            try:
+                self.cfnode.set_enable(1)
+                self.shell.log.info("The Referral has been enabled.")
+            except Exception as e:
+                raise configshell.ExecutionError(
+                    "The Referral could not be enabled.")
+
+    def ui_command_disable(self):
+        '''
+        Disables the current Referral.
+
+        SEE ALSO
+        ========
+        B{enable}
+        '''
+        if not self.cfnode.get_enable():
+            self.shell.log.info("The Referral is already disabled.")
+        else:
+            try:
+                self.cfnode.set_enable(0)
+                self.shell.log.info("The Referral has been disabled.")
+            except Exception as e:
+                raise configshell.ExecutionError(
+                    "The Referral could not be disabled.")
+
+
+class UIANAGroupsNode(UINode):
+    def __init__(self, parent):
+        UINode.__init__(self, 'ana_groups', parent)
+
+    def refresh(self):
+        self._children = set([])
+        for a in self.parent.cfnode.ana_groups:
+            UIANAGroupNode(self, a)
+
+    def ui_command_create(self, grpid):
+        '''
+        Creates a new ANA Group.
+
+        SEE ALSO
+        ========
+        B{delete}
+        '''
+        a = nvme.ANAGroup(self.parent.cfnode, grpid, mode='create')
+        UIANAGroupNode(self, a)
+
+    def ui_command_delete(self, grpid):
+        '''
+        Deletes the ANA Group with the specified I{name}.
+
+        SEE ALSO
+        ========
+        B{create}
+        '''
+        a = nvme.ANAGroup(self.parent.cfnode, grpid, mode='lookup')
+        a.delete()
+        self.refresh()
+
+
+class UIANAGroupNode(UINode):
+    ui_desc_ana = {
+        'state' : ('string', 'ANA state'),
+    }
+
+    def __init__(self, parent, cfnode):
+        UINode.__init__(self, str(cfnode.grpid), parent, cfnode)
+
+    def summary(self):
+        info = []
+        info.append("state=" + self.cfnode.get_attr("ana", "state"))
+        return (", ".join(info), True)
+
+class UIHostsNode(UINode):
+    def __init__(self, parent):
+        UINode.__init__(self, 'hosts', parent)
+
+    def refresh(self):
+        self._children = set([])
+        for host in self.parent.cfnode.hosts:
+            UIHostNode(self, host)
+
+    def ui_command_create(self, nqn):
+        '''
+        Creates a new NVMe host.
+
+        SEE ALSO
+        ========
+        B{delete}
+        '''
+        host = nvme.Host(nqn, mode='create')
+        UIHostNode(self, host)
+
+    def ui_command_delete(self, nqn):
+        '''
+        Recursively deletes the NVMe Host with the specified I{nqn}, and all
+        objects hanging under it.
+
+        SEE ALSO
+        ========
+        B{create}
+        '''
+        host = nvme.Host(nqn, mode='lookup')
+        host.delete()
+        self.refresh()
+
+
+class UIHostNode(UINode):
+    def __init__(self, parent, cfnode):
+        UINode.__init__(self, cfnode.nqn, parent, cfnode)
+
+
+def usage():
+    print("syntax: %s save [file_to_save_to]" % sys.argv[0])
+    print("        %s restore [file_to_restore_from]" % sys.argv[0])
+    print("        %s clear" % sys.argv[0])
+    print("        %s ls" % sys.argv[0])
+    sys.exit(-1)
+
+
+def save(to_file):
+    nvme.Root().save_to_file(to_file)
+
+
+def restore(from_file):
+    errors = None
+
+    try:
+        errors = nvme.Root().restore_from_file(from_file)
+    except IOError as e:
+        if not from_file:
+            from_file = nvme.DEFAULT_SAVE_FILE
+
+        if e.errno == errno.ENOENT:
+            # Not an error if the restore file is not present
+            print("No saved config file at %s, ok, exiting" % from_file)
+            sys.exit(0)
+        else:
+            print("Error processing config file at %s, error %s, exiting" %
+                  (from_file, str(e)))
+            sys.exit(1)
+
+    # These errors are non-fatal
+    for error in errors:
+        print(error)
+
+    sys.exit(0)
+
+
+def clear(unused):
+    nvme.Root().clear_existing()
+
+
+def ls(unused):
+    shell = configshell.shell.ConfigShell('~/.nvmetcli')
+    UIRootNode(shell)
+    shell.run_cmdline("ls")
+    sys.exit(0)
+
+
+funcs = dict(save=save, restore=restore, clear=clear, ls=ls)
+
+
+def main():
+    if os.geteuid() != 0:
+        print("%s: must run as root." % sys.argv[0], file=sys.stderr)
+        sys.exit(-1)
+
+    if len(sys.argv) > 3:
+        usage()
+
+    if len(sys.argv) == 2 or len(sys.argv) == 3:
+        if sys.argv[1] == "--help":
+            usage()
+
+        if sys.argv[1] not in funcs.keys():
+            usage()
+
+        if len(sys.argv) == 3:
+            savefile = sys.argv[2]
+        else:
+            savefile = None
+
+        funcs[sys.argv[1]](savefile)
+        return
+
+    try:
+        shell = configshell.shell.ConfigShell('~/.nvmetcli')
+        UIRootNode(shell)
+    except Exception as msg:
+        shell.log.error(str(msg))
+        return
+
+    while not shell._exit:
+        try:
+            shell.run_interactive()
+        except Exception as msg:
+            shell.log.error(str(msg))
+
+
+if __name__ == "__main__":
+    main()