diff options
Diffstat (limited to '')
| -rw-r--r-- | fabrics.c | 2554 |
1 files changed, 944 insertions, 1610 deletions
@@ -27,7 +27,6 @@ #include <stdint.h> #include <unistd.h> #include <dirent.h> -#include <sys/ioctl.h> #include <inttypes.h> #include <libgen.h> #include <sys/stat.h> @@ -36,720 +35,231 @@ #include <time.h> #include <sys/types.h> -#include <arpa/inet.h> -#include <netdb.h> - -#include "util/parser.h" -#include "nvme-ioctl.h" -#include "nvme-status.h" -#include "fabrics.h" +#include <linux/types.h> +#include "common.h" #include "nvme.h" -#include "util/argconfig.h" +#include "libnvme.h" +#include "nvme-print.h" -#include "common.h" -#include "util/log.h" -#include "util/cleanup.h" - -#ifdef HAVE_SYSTEMD -#include <systemd/sd-id128.h> -#define NVME_HOSTNQN_ID SD_ID128_MAKE(c7,f4,61,81,12,be,49,32,8c,83,10,6f,9d,dd,d8,6b) -#endif - -#define NVMF_HOSTID_SIZE 36 - -/* default to 600 seconds of reconnect attempts before giving up */ -#define NVMF_DEF_CTRL_LOSS_TMO 600 - -const char *conarg_nqn = "nqn"; -const char *conarg_transport = "transport"; -const char *conarg_traddr = "traddr"; -const char *conarg_trsvcid = "trsvcid"; -const char *conarg_host_traddr = "host_traddr"; -const char *conarg_host_iface = "host_iface"; - -struct fabrics_config fabrics_cfg = { - .ctrl_loss_tmo = NVMF_DEF_CTRL_LOSS_TMO, - .fast_io_fail_tmo = -1, - .output_format = "normal", -}; +#define PATH_NVMF_DISC SYSCONFDIR "/nvme/discovery.conf" +#define PATH_NVMF_CONFIG SYSCONFDIR "/nvme/config.json" +#define MAX_DISC_ARGS 32 +#define MAX_DISC_RETRIES 10 -struct connect_args { +#define NVMF_DEF_DISC_TMO 30 + +/* Name of file to output log pages in their raw format */ +static char *raw; +static bool persistent; +static bool quiet; +static bool dump_config; + +static const char *nvmf_tport = "transport type"; +static const char *nvmf_traddr = "transport address"; +static const char *nvmf_nqn = "subsystem nqn"; +static const char *nvmf_trsvcid = "transport service id (e.g. IP port)"; +static const char *nvmf_htraddr = "host traddr (e.g. FC WWN's)"; +static const char *nvmf_hiface = "host interface (for tcp transport)"; +static const char *nvmf_hostnqn = "user-defined hostnqn"; +static const char *nvmf_hostid = "user-defined hostid (if default not used)"; +static const char *nvmf_hostkey = "user-defined dhchap key (if default not used)"; +static const char *nvmf_ctrlkey = "user-defined dhchap controller key (for bi-directional authentication)"; +static const char *nvmf_nr_io_queues = "number of io queues to use (default is core count)"; +static const char *nvmf_nr_write_queues = "number of write queues to use (default 0)"; +static const char *nvmf_nr_poll_queues = "number of poll queues to use (default 0)"; +static const char *nvmf_queue_size = "number of io queue elements to use (default 128)"; +static const char *nvmf_keep_alive_tmo = "keep alive timeout period in seconds"; +static const char *nvmf_reconnect_delay = "reconnect timeout period in seconds"; +static const char *nvmf_ctrl_loss_tmo = "controller loss timeout period in seconds"; +static const char *nvmf_tos = "type of service"; +static const char *nvmf_dup_connect = "allow duplicate connections between same transport host and subsystem port"; +static const char *nvmf_disable_sqflow = "disable controller sq flow control (default false)"; +static const char *nvmf_hdr_digest = "enable transport protocol header digest (TCP transport)"; +static const char *nvmf_data_digest = "enable transport protocol data digest (TCP transport)"; +static const char *nvmf_config_file = "Use specified JSON configuration file or 'none' to disable"; + +#define NVMF_OPTS(c) \ + OPT_STRING("transport", 't', "STR", &transport, nvmf_tport), \ + OPT_STRING("traddr", 'a', "STR", &traddr, nvmf_traddr), \ + OPT_STRING("trsvcid", 's', "STR", &trsvcid, nvmf_trsvcid), \ + OPT_STRING("host-traddr", 'w', "STR", &c.host_traddr, nvmf_htraddr), \ + OPT_STRING("host-iface", 'f', "STR", &c.host_iface, nvmf_hiface), \ + OPT_STRING("hostnqn", 'q', "STR", &hostnqn, nvmf_hostnqn), \ + OPT_STRING("hostid", 'I', "STR", &hostid, nvmf_hostid), \ + OPT_STRING("nqn", 'n', "STR", &subsysnqn, nvmf_nqn), \ + OPT_STRING("dhchap-secret", 'S', "STR", &hostkey, nvmf_hostkey), \ + OPT_STRING("dhchap-ctrl-secret", 'C', "STR", &ctrlkey, nvmf_ctrlkey), \ + OPT_INT("nr-io-queues", 'i', &c.nr_io_queues, nvmf_nr_io_queues), \ + OPT_INT("nr-write-queues", 'W', &c.nr_write_queues, nvmf_nr_write_queues),\ + OPT_INT("nr-poll-queues", 'P', &c.nr_poll_queues, nvmf_nr_poll_queues), \ + OPT_INT("queue-size", 'Q', &c.queue_size, nvmf_queue_size), \ + OPT_INT("keep-alive-tmo", 'k', &c.keep_alive_tmo, nvmf_keep_alive_tmo), \ + OPT_INT("reconnect-delay", 'c', &c.reconnect_delay, nvmf_reconnect_delay),\ + OPT_INT("ctrl-loss-tmo", 'l', &c.ctrl_loss_tmo, nvmf_ctrl_loss_tmo), \ + OPT_INT("tos", 'T', &c.tos, nvmf_tos), \ + OPT_FLAG("duplicate-connect", 'D', &c.duplicate_connect, nvmf_dup_connect), \ + OPT_FLAG("disable-sqflow", 'd', &c.disable_sqflow, nvmf_disable_sqflow), \ + OPT_FLAG("hdr-digest", 'g', &c.hdr_digest, nvmf_hdr_digest), \ + OPT_FLAG("data-digest", 'G', &c.data_digest, nvmf_data_digest) \ + +struct tr_config { char *subsysnqn; char *transport; char *traddr; - char *trsvcid; char *host_traddr; char *host_iface; - struct connect_args *next; - struct connect_args *tail; -}; - -struct connect_args *tracked_ctrls; - -#define PATH_NVME_FABRICS "/dev/nvme-fabrics" -#define PATH_NVMF_DISC "/etc/nvme/discovery.conf" -#define PATH_NVMF_HOSTNQN "/etc/nvme/hostnqn" -#define PATH_NVMF_HOSTID "/etc/nvme/hostid" -#define MAX_DISC_ARGS 10 -#define MAX_DISC_RETRIES 10 - -enum { - OPT_INSTANCE, - OPT_CNTLID, - OPT_ERR -}; - -static const match_table_t opt_tokens = { - { OPT_INSTANCE, "instance=%d" }, - { OPT_CNTLID, "cntlid=%d" }, - { OPT_ERR, NULL }, -}; - -const char *arg_str(const char * const *strings, - size_t array_size, size_t idx) -{ - if (idx < array_size && strings[idx]) - return strings[idx]; - return "unrecognized"; -} - -const char * const trtypes[] = { - [NVMF_TRTYPE_RDMA] = "rdma", - [NVMF_TRTYPE_FC] = "fc", - [NVMF_TRTYPE_TCP] = "tcp", - [NVMF_TRTYPE_LOOP] = "loop", -}; - -static const char *trtype_str(__u8 trtype) -{ - return arg_str(trtypes, ARRAY_SIZE(trtypes), trtype); -} - -static const char * const adrfams[] = { - [NVMF_ADDR_FAMILY_PCI] = "pci", - [NVMF_ADDR_FAMILY_IP4] = "ipv4", - [NVMF_ADDR_FAMILY_IP6] = "ipv6", - [NVMF_ADDR_FAMILY_IB] = "infiniband", - [NVMF_ADDR_FAMILY_FC] = "fibre-channel", - [NVMF_ADDR_FAMILY_LOOP] = "loop", -}; - -static inline const char *adrfam_str(__u8 adrfam) -{ - return arg_str(adrfams, ARRAY_SIZE(adrfams), adrfam); -} - -static const char * const subtypes[] = { - [NVME_NQN_DISC] = "discovery subsystem", - [NVME_NQN_NVME] = "nvme subsystem", -}; - -static inline const char *subtype_str(__u8 subtype) -{ - return arg_str(subtypes, ARRAY_SIZE(subtypes), subtype); -} - -static const char * const treqs[] = { - [NVMF_TREQ_NOT_SPECIFIED] = "not specified", - [NVMF_TREQ_REQUIRED] = "required", - [NVMF_TREQ_NOT_REQUIRED] = "not required", - [NVMF_TREQ_DISABLE_SQFLOW] = "not specified, " - "sq flow control disable supported", -}; - -static inline const char *treq_str(__u8 treq) -{ - return arg_str(treqs, ARRAY_SIZE(treqs), treq); -} - -static const char * const sectypes[] = { - [NVMF_TCP_SECTYPE_NONE] = "none", - [NVMF_TCP_SECTYPE_TLS] = "tls", -}; - -static inline const char *sectype_str(__u8 sectype) -{ - return arg_str(sectypes, ARRAY_SIZE(sectypes), sectype); -} - -static const char * const prtypes[] = { - [NVMF_RDMA_PRTYPE_NOT_SPECIFIED] = "not specified", - [NVMF_RDMA_PRTYPE_IB] = "infiniband", - [NVMF_RDMA_PRTYPE_ROCE] = "roce", - [NVMF_RDMA_PRTYPE_ROCEV2] = "roce-v2", - [NVMF_RDMA_PRTYPE_IWARP] = "iwarp", -}; - -static inline const char *prtype_str(__u8 prtype) -{ - return arg_str(prtypes, ARRAY_SIZE(prtypes), prtype); -} - -static const char * const qptypes[] = { - [NVMF_RDMA_QPTYPE_CONNECTED] = "connected", - [NVMF_RDMA_QPTYPE_DATAGRAM] = "datagram", -}; - -static inline const char *qptype_str(__u8 qptype) -{ - return arg_str(qptypes, ARRAY_SIZE(qptypes), qptype); -} - -static const char * const cms[] = { - [NVMF_RDMA_CMS_RDMA_CM] = "rdma-cm", + char *trsvcid; }; -static const char *cms_str(__u8 cm) +static void space_strip_len(int max, char *str) { - return arg_str(cms, ARRAY_SIZE(cms), cm); -} - -/* - * parse strings with connect arguments to find a particular field. - * If field found, return string containing field value. If field - * not found, return an empty string. - */ -char *parse_conn_arg(const char *conargs, const char delim, const char *field) -{ - char *s, *e; - size_t cnt; - - /* - * There are field name overlaps: traddr and host_traddr. - * By chance, both connect arg strings are set up to - * have traddr field followed by host_traddr field. Thus field - * name matching doesn't overlap in the searches. Technically, - * as is, the loop and delimiter checking isn't necessary. - * However, better to be prepared. - */ - do { - s = strstr(conargs, field); - if (!s) - goto empty_field; - /* validate prior character is delimiter */ - if (s == conargs || *(s - 1) == delim) { - /* match requires next character to be assignment */ - s += strlen(field); - if (*s == '=') - /* match */ - break; - } - /* field overlap: seek to delimiter and keep looking */ - conargs = strchr(s, delim); - if (!conargs) - goto empty_field; - conargs++; /* skip delimiter */ - } while (1); - s++; /* skip assignment character */ - e = strchr(s, delim); - if (e) - cnt = e - s; - else - cnt = strlen(s); - - return strndup(s, cnt); - -empty_field: - return strdup("\0"); -} - -int ctrl_instance(const char *device) -{ - char d[64]; - const char *p; - int ret, instance; - - p = strrchr(device, '/'); - if (p == NULL) - p = device; - else - p++; - ret = sscanf(p, "nvme%d", &instance); - if (ret <= 0) - return -EINVAL; - if (snprintf(d, sizeof(d), "nvme%d", instance) <= 0 || - strcmp(p, d)) - return -EINVAL; - return instance; -} - -/* - * Given a controller name, create a connect_args with its - * attributes and compare the attributes against the connect args - * given. - * Return true/false based on whether it matches - */ -static bool ctrl_matches_connectargs(const char *name, struct connect_args *args) -{ - struct connect_args cargs; - bool found = false; - char *path = NULL, *addr; - int ret; - bool persistent = true; - - ret = asprintf(&path, "%s/%s", SYS_NVME, name); - if (ret < 0) - return found; - - addr = nvme_get_ctrl_attr(path, "address"); - cargs.subsysnqn = nvme_get_ctrl_attr(path, "subsysnqn"); - cargs.transport = nvme_get_ctrl_attr(path, "transport"); - - if (!addr || !cargs.subsysnqn || !cargs.transport) { - fprintf(stderr, "nvme_get_ctrl_attr failed\n"); - goto out; - } + int i; - cargs.traddr = parse_conn_arg(addr, ' ', conarg_traddr); - cargs.trsvcid = parse_conn_arg(addr, ' ', conarg_trsvcid); - cargs.host_traddr = parse_conn_arg(addr, ' ', conarg_host_traddr); - cargs.host_iface = parse_conn_arg(addr, ' ', conarg_host_iface); - - if (!strcmp(cargs.subsysnqn, NVME_DISC_SUBSYS_NAME)) { - char *kato_str = nvme_get_ctrl_attr(path, "kato"), *p; - unsigned int kato = 0; - - /* - * When looking up discovery controllers we have to skip - * any non-persistent controllers (ie those with a zero - * kato value). Otherwise the controller will vanish from - * underneath us as they are owned by another program. - * - * On older kernels, the 'kato' attribute isn't present. - * Assume a persistent controller for these installations. - */ - if (kato_str) { - kato = strtoul(kato_str, &p, 0); - if (p == kato_str) - kato = 0; - free(kato_str); - persistent = (kato != 0); - } + for (i = max - 1; i >= 0; i--) { + if (str[i] != '\0' && str[i] != ' ') + return; + else + str[i] = '\0'; } - - if (persistent && - !strcmp(cargs.subsysnqn, args->subsysnqn) && - !strcmp(cargs.transport, args->transport) && - (!strcmp(cargs.traddr, args->traddr) || - !strcmp(args->traddr, "none")) && - (!strcmp(cargs.trsvcid, args->trsvcid) || - !strcmp(args->trsvcid, "none")) && - (!strcmp(cargs.host_traddr, args->host_traddr) || - !strcmp(args->host_traddr, "none")) && - (!strcmp(cargs.host_iface, args->host_iface) || - !strcmp(args->host_iface, "none"))) - found = true; - - free(cargs.traddr); - free(cargs.trsvcid); - free(cargs.host_traddr); - free(cargs.host_iface); -out: - free(cargs.subsysnqn); - free(cargs.transport); - free(addr); - free(path); - - return found; } -/* - * Look through the system to find an existing controller whose - * attributes match the connect arguments specified - * If found, a string containing the controller name (ex: "nvme?") - * is returned. - * If not found, a NULL is returned. - */ -static char *find_ctrl_with_connectargs(struct connect_args *args) +static int set_discovery_kato(struct nvme_fabrics_config *cfg) { - struct dirent **devices; - char *devname = NULL; - int i, n; - - n = scandir(SYS_NVME, &devices, scan_ctrls_filter, alphasort); - if (n < 0) { - msg(LOG_ERR, "no NVMe controller(s) detected.\n"); - return NULL; - } + int tmo = cfg->keep_alive_tmo; - for (i = 0; i < n; i++) { - if (ctrl_matches_connectargs(devices[i]->d_name, args)) { - devname = strdup(devices[i]->d_name); - if (devname == NULL) - msg(LOG_ERR, "no memory for ctrl name %s\n", - devices[i]->d_name); - goto cleanup_devices; - } - } - -cleanup_devices: - for (i = 0; i < n; i++) - free(devices[i]); - free(devices); + /* Set kato to NVMF_DEF_DISC_TMO for persistent controllers */ + if (persistent && !cfg->keep_alive_tmo) + cfg->keep_alive_tmo = NVMF_DEF_DISC_TMO; + /* Set kato to zero for non-persistent controllers */ + else if (!persistent && (cfg->keep_alive_tmo > 0)) + cfg->keep_alive_tmo = 0; - return devname; + return tmo; } -static struct connect_args *extract_connect_args(char *argstr) +static nvme_ctrl_t __create_discover_ctrl(nvme_root_t r, nvme_host_t h, + struct nvme_fabrics_config *cfg, + struct tr_config *trcfg) { - struct connect_args *cargs; + nvme_ctrl_t c; + int tmo, ret; - cargs = calloc(1, sizeof(*cargs)); - if (!cargs) + c = nvme_create_ctrl(r, trcfg->subsysnqn, trcfg->transport, + trcfg->traddr, trcfg->host_traddr, + trcfg->host_iface, trcfg->trsvcid); + if (!c) return NULL; - cargs->subsysnqn = parse_conn_arg(argstr, ',', conarg_nqn); - cargs->transport = parse_conn_arg(argstr, ',', conarg_transport); - cargs->traddr = parse_conn_arg(argstr, ',', conarg_traddr); - cargs->trsvcid = parse_conn_arg(argstr, ',', conarg_trsvcid); - cargs->host_traddr = parse_conn_arg(argstr, ',', conarg_host_traddr); - cargs->host_iface = parse_conn_arg(argstr, ',', conarg_host_iface); - return cargs; -} - -static void destruct_connect_args(struct connect_args *cargs) -{ - free(cargs->subsysnqn); - free(cargs->transport); - free(cargs->traddr); - free(cargs->trsvcid); - free(cargs->host_traddr); - free(cargs->host_iface); -} - -static void free_connect_args(struct connect_args *cargs) -{ - destruct_connect_args(cargs); - free(cargs); -} - -static void track_ctrl(char *argstr) -{ - struct connect_args *cargs; - - cargs = extract_connect_args(argstr); - if (!cargs) - return; - - if (!tracked_ctrls) - tracked_ctrls = cargs; - else - tracked_ctrls->tail->next = cargs; - tracked_ctrls->tail = cargs; -} - -static int add_ctrl(const char *argstr) -{ - substring_t args[MAX_OPT_ARGS]; - char buf[BUF_SIZE], *options, *p; - int token, ret, fd, len = strlen(argstr); - - fd = open(PATH_NVME_FABRICS, O_RDWR); - if (fd < 0) { - msg(LOG_ERR, "Failed to open %s: %s\n", - PATH_NVME_FABRICS, strerror(errno)); - ret = -errno; - goto out; - } - - ret = write(fd, argstr, len); - if (ret != len) { - if (errno != EALREADY) - msg(LOG_NOTICE, "Failed to write to %s: %s\n", - PATH_NVME_FABRICS, strerror(errno)); - ret = -errno; - goto out_close; - } - - len = read(fd, buf, BUF_SIZE); - if (len < 0) { - msg(LOG_ERR, "Failed to read from %s: %s\n", - PATH_NVME_FABRICS, strerror(errno)); - ret = -errno; - goto out_close; - } - - buf[len] = '\0'; - options = buf; - while ((p = strsep(&options, ",\n")) != NULL) { - if (!*p) - continue; - - token = match_token(p, opt_tokens, args); - switch (token) { - case OPT_INSTANCE: - if (match_int(args, &token)) - goto out_fail; - ret = token; - track_ctrl((char *)argstr); - goto out_close; - default: - /* ignore */ - break; - } - } - -out_fail: - msg(LOG_ERR, "Failed to parse ctrl info for \"%s\"\n", argstr); - ret = -EINVAL; -out_close: - close(fd); -out: - return ret; -} - -static int remove_ctrl_by_path(char *sysfs_path) -{ - int ret, fd; - - fd = open(sysfs_path, O_WRONLY); - if (fd < 0) { - ret = -errno; - msg(LOG_ERR, "Failed to open %s: %s\n", sysfs_path, - strerror(errno)); - goto out; - } - if (write(fd, "1", 1) != 1) { - ret = -errno; - goto out_close; - } + nvme_ctrl_set_discovery_ctrl(c, true); + tmo = set_discovery_kato(cfg); - ret = 0; -out_close: - close(fd); -out: - return ret; -} + errno = 0; + ret = nvmf_add_ctrl(h, c, cfg); -int remove_ctrl(int instance) -{ - char *sysfs_path; - int ret; - - if (asprintf(&sysfs_path, "/sys/class/nvme/nvme%d/delete_controller", - instance) < 0) { - ret = -errno; - goto out; + cfg->keep_alive_tmo = tmo; + if (ret) { + errno = ret; + nvme_free_ctrl(c); + return NULL; } - ret = remove_ctrl_by_path(sysfs_path); - free(sysfs_path); -out: - return ret; + return c; } -enum { - DISC_OK, - DISC_NO_LOG, - DISC_GET_NUMRECS, - DISC_GET_LOG, - DISC_RETRY_EXHAUSTED, - DISC_NOT_EQUAL, -}; - -static int nvmf_get_log_page_discovery(const char *dev_path, - struct nvmf_disc_rsp_page_hdr **logp, int *numrec, int *status) +static nvme_ctrl_t create_discover_ctrl(nvme_root_t r, nvme_host_t h, + struct nvme_fabrics_config *cfg, + struct tr_config *trcfg) { - struct nvmf_disc_rsp_page_hdr *log; - unsigned int hdr_size; - unsigned long genctr; - int error, fd, max_retries = MAX_DISC_RETRIES, retries = 0; - - fd = open(dev_path, O_RDWR); - if (fd < 0) { - error = -errno; - msg(LOG_ERR, "Failed to open %s: %s\n", - dev_path, strerror(errno)); - goto out; - } - - /* first get_log_page we just need numrec entry from discovery hdr. - * host supplies its desired bytes via dwords, per NVMe spec. - */ - hdr_size = round_up((offsetof(struct nvmf_disc_rsp_page_hdr, numrec) + - sizeof(log->numrec)), sizeof(__u32)); - - /* - * Issue first get log page w/numdl small enough to retrieve numrec. - * We just want to know how many records to retrieve. - */ - log = calloc(1, hdr_size); - if (!log) { - perror("could not alloc memory for discovery log header"); - error = -ENOMEM; - goto out_close; - } - - error = nvme_discovery_log(fd, log, hdr_size); - if (error) { - error = DISC_GET_NUMRECS; - goto out_free_log; + nvme_ctrl_t c; + + c = __create_discover_ctrl(r, h, cfg, trcfg); + if (!persistent) + return c; + + /* Find out the name of discovery controller */ + struct nvme_id_ctrl id = { 0 }; + if (nvme_ctrl_identify(c, &id)) { + fprintf(stderr, "failed to identify controller, error %s\n", + nvme_strerror(errno)); + nvme_disconnect_ctrl(c); + nvme_free_ctrl(c); + return NULL; } - do { - unsigned int log_size; - - /* check numrec limits */ - *numrec = le64_to_cpu(log->numrec); - genctr = le64_to_cpu(log->genctr); - free(log); - - if (*numrec == 0) { - error = DISC_NO_LOG; - goto out_close; - } - - /* we are actually retrieving the entire discovery tables - * for the second get_log_page(), per - * NVMe spec so no need to round_up(), or there is something - * seriously wrong with the standard - */ - log_size = sizeof(struct nvmf_disc_rsp_page_hdr) + - sizeof(struct nvmf_disc_rsp_page_entry) * *numrec; - - /* allocate discovery log pages based on page_hdr->numrec */ - log = calloc(1, log_size); - if (!log) { - perror("could not alloc memory for discovery log page"); - error = -ENOMEM; - goto out_close; - } - - /* - * issue new get_log_page w/numdl+numdh set to get all records, - * up to MAX_DISC_LOGS. - */ - error = nvme_discovery_log(fd, log, log_size); - if (error) { - error = DISC_GET_LOG; - goto out_free_log; - } - - /* - * The above call to nvme_discovery_log() might result - * in several calls (with different offsets), so we need - * to fetch the header again to have the most up-to-date - * value for the generation counter - */ - genctr = le64_to_cpu(log->genctr); - error = nvme_discovery_log(fd, log, hdr_size); - if (error) { - error = DISC_GET_LOG; - goto out_free_log; - } - } while (genctr != le64_to_cpu(log->genctr) && - ++retries < max_retries); + if (!strcmp(id.subnqn, NVME_DISC_SUBSYS_NAME)) + return c; /* - * If genctr is still different with the one in the log entry, it - * means the retires have been exhausted to max_retries. Then it - * should be retried by the caller or the user. + * The subsysnqn is not the well-known name. Prefer the unique + * subsysnqn over the well-known one. */ - if (genctr != le64_to_cpu(log->genctr)) { - error = DISC_RETRY_EXHAUSTED; - goto out_free_log; - } - - if (*numrec != le64_to_cpu(log->numrec)) { - error = DISC_NOT_EQUAL; - goto out_free_log; - } - - /* needs to be freed by the caller */ - *logp = log; - error = DISC_OK; - goto out_close; - -out_free_log: - free(log); -out_close: - close(fd); -out: - *status = nvme_status_to_errno(error, true); - return error; -} + nvme_disconnect_ctrl(c); + nvme_free_ctrl(c); -static int space_strip_len(int max, const char *str) -{ - int i; - - for (i = max - 1; i >= 0; i--) - if (str[i] != '\0' && str[i] != ' ') - break; - - return i + 1; + trcfg->subsysnqn = id.subnqn; + return __create_discover_ctrl(r, h, cfg, trcfg); } -static void print_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec, - int instance) +static void print_discovery_log(struct nvmf_discovery_log *log, int numrec) { int i; - printf("\n"); - - if (fabrics_cfg.persistent) - printf("Persistent device: nvme%d\n", instance); - - printf("Discovery Log Number of Records %d, " + printf("\nDiscovery Log Number of Records %d, " "Generation counter %"PRIu64"\n", numrec, le64_to_cpu(log->genctr)); for (i = 0; i < numrec; i++) { - struct nvmf_disc_rsp_page_entry *e = &log->entries[i]; + struct nvmf_disc_log_entry *e = &log->entries[i]; + + space_strip_len(NVMF_TRSVCID_SIZE, e->trsvcid); + space_strip_len(NVMF_TRADDR_SIZE, e->traddr); printf("=====Discovery Log Entry %d======\n", i); - printf("trtype: %s\n", trtype_str(e->trtype)); - printf("adrfam: %s\n", adrfam_str(e->adrfam)); - printf("subtype: %s\n", subtype_str(e->subtype)); - printf("treq: %s\n", treq_str(e->treq)); + printf("trtype: %s\n", nvmf_trtype_str(e->trtype)); + printf("adrfam: %s\n", + e->traddr && strlen(e->traddr) ? + nvmf_adrfam_str(e->adrfam): ""); + printf("subtype: %s\n", nvmf_subtype_str(e->subtype)); + printf("treq: %s\n", nvmf_treq_str(e->treq)); printf("portid: %d\n", e->portid); - printf("trsvcid: %.*s\n", - space_strip_len(NVMF_TRSVCID_SIZE, e->trsvcid), - e->trsvcid); + printf("trsvcid: %s\n", e->trsvcid); printf("subnqn: %s\n", e->subnqn); - printf("traddr: %.*s\n", - space_strip_len(NVMF_TRADDR_SIZE, e->traddr), - e->traddr); + printf("traddr: %s\n", e->traddr); + printf("eflags: %s\n", nvmf_eflags_str(e->eflags)); switch (e->trtype) { case NVMF_TRTYPE_RDMA: printf("rdma_prtype: %s\n", - prtype_str(e->tsas.rdma.prtype)); + nvmf_prtype_str(e->tsas.rdma.prtype)); printf("rdma_qptype: %s\n", - qptype_str(e->tsas.rdma.qptype)); + nvmf_qptype_str(e->tsas.rdma.qptype)); printf("rdma_cms: %s\n", - cms_str(e->tsas.rdma.cms)); + nvmf_cms_str(e->tsas.rdma.cms)); printf("rdma_pkey: 0x%04x\n", le16_to_cpu(e->tsas.rdma.pkey)); break; case NVMF_TRTYPE_TCP: printf("sectype: %s\n", - sectype_str(e->tsas.tcp.sectype)); + nvmf_sectype_str(e->tsas.tcp.sectype)); break; } } } -static void json_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec, - int instance) +static void json_discovery_log(struct nvmf_discovery_log *log, int numrec) { struct json_object *root; struct json_object *entries; - char *dev_name = NULL; int i; - if (asprintf(&dev_name, "nvme%d", instance) < 0) - return; - root = json_create_object(); entries = json_create_array(); - json_object_add_value_string(root, "device", dev_name); - json_object_add_value_uint(root, "genctr", le64_to_cpu(log->genctr)); + json_object_add_value_uint64(root, "genctr", le64_to_cpu(log->genctr)); json_object_add_value_array(root, "records", entries); for (i = 0; i < numrec; i++) { - struct nvmf_disc_rsp_page_entry *e = &log->entries[i]; + struct nvmf_disc_log_entry *e = &log->entries[i]; struct json_object *entry = json_create_object(); nvme_strip_spaces(e->trsvcid, NVMF_TRSVCID_SIZE); @@ -757,32 +267,34 @@ static void json_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec, nvme_strip_spaces(e->traddr, NVMF_TRADDR_SIZE); json_object_add_value_string(entry, "trtype", - trtype_str(e->trtype)); + nvmf_trtype_str(e->trtype)); json_object_add_value_string(entry, "adrfam", - adrfam_str(e->adrfam)); + nvmf_adrfam_str(e->adrfam)); json_object_add_value_string(entry, "subtype", - subtype_str(e->subtype)); + nvmf_subtype_str(e->subtype)); json_object_add_value_string(entry,"treq", - treq_str(e->treq)); - json_object_add_value_uint(entry, "portid", e->portid); + nvmf_treq_str(e->treq)); + json_object_add_value_uint(entry, "portid", + le16_to_cpu(e->portid)); json_object_add_value_string(entry, "trsvcid", e->trsvcid); json_object_add_value_string(entry, "subnqn", e->subnqn); json_object_add_value_string(entry, "traddr", e->traddr); + json_object_add_value_uint(entry, "eflags", e->eflags); switch (e->trtype) { case NVMF_TRTYPE_RDMA: json_object_add_value_string(entry, "rdma_prtype", - prtype_str(e->tsas.rdma.prtype)); + nvmf_prtype_str(e->tsas.rdma.prtype)); json_object_add_value_string(entry, "rdma_qptype", - qptype_str(e->tsas.rdma.qptype)); + nvmf_qptype_str(e->tsas.rdma.qptype)); json_object_add_value_string(entry, "rdma_cms", - cms_str(e->tsas.rdma.cms)); + nvmf_cms_str(e->tsas.rdma.cms)); json_object_add_value_uint(entry, "rdma_pkey", le16_to_cpu(e->tsas.rdma.pkey)); break; case NVMF_TRTYPE_TCP: json_object_add_value_string(entry, "sectype", - sectype_str(e->tsas.tcp.sectype)); + nvmf_sectype_str(e->tsas.tcp.sectype)); break; } json_array_add_value_object(entries, entry); @@ -790,1158 +302,980 @@ static void json_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec, json_print_object(root, NULL); printf("\n"); json_free_object(root); - free(dev_name); } -static void save_discovery_log(struct nvmf_disc_rsp_page_hdr *log, int numrec) +static void save_discovery_log(char *raw, struct nvmf_discovery_log *log) { - int fd; - int len, ret; + uint64_t numrec = le64_to_cpu(log->numrec); + int fd, len, ret; - fd = open(fabrics_cfg.raw, O_CREAT|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR); + fd = open(raw, O_CREAT|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR); if (fd < 0) { - msg(LOG_ERR, "failed to open %s: %s\n", - fabrics_cfg.raw, strerror(errno)); + fprintf(stderr, "failed to open %s: %s\n", + raw, strerror(errno)); return; } - len = sizeof(struct nvmf_disc_rsp_page_hdr) + - numrec * sizeof(struct nvmf_disc_rsp_page_entry); + len = sizeof(struct nvmf_discovery_log) + + numrec * sizeof(struct nvmf_disc_log_entry); ret = write(fd, log, len); if (ret < 0) - msg(LOG_ERR, "failed to write to %s: %s\n", - fabrics_cfg.raw, strerror(errno)); + fprintf(stderr, "failed to write to %s: %s\n", + raw, strerror(errno)); else - printf("Discovery log is saved to %s\n", fabrics_cfg.raw); + printf("Discovery log is saved to %s\n", raw); close(fd); } -static void print_connect_msg(int instance) +static void print_connect_msg(nvme_ctrl_t c) { - printf("device: nvme%d\n", instance); + printf("device: %s\n", nvme_ctrl_get_name(c)); } -static void json_connect_msg(int instance) +static void json_connect_msg(nvme_ctrl_t c) { struct json_object *root; - char *dev_name = NULL; - - if (asprintf(&dev_name, "nvme%d", instance) < 0) - return; root = json_create_object(); - json_object_add_value_string(root, "device", dev_name); + json_object_add_value_string(root, "device", nvme_ctrl_get_name(c)); json_print_object(root, NULL); printf("\n"); json_free_object(root); - free(dev_name); } -static char *hostnqn_read_file(void) +static int __discover(nvme_ctrl_t c, struct nvme_fabrics_config *defcfg, + char *raw, bool connect, bool persistent, + enum nvme_print_flags flags) { - FILE *f; - char hostnqn[NVMF_NQN_SIZE]; - char *ret = NULL; - - f = fopen(PATH_NVMF_HOSTNQN, "r"); - if (f == NULL) - return false; - - if (fgets(hostnqn, sizeof(hostnqn), f) == NULL || - !strlen(hostnqn)) - goto out; + struct nvmf_discovery_log *log = NULL; + nvme_subsystem_t s = nvme_ctrl_get_subsystem(c); + nvme_host_t h = nvme_subsystem_get_host(s); + uint64_t numrec; + int err; - ret = strndup(hostnqn, strcspn(hostnqn, "\n")); -out: - fclose(f); - if (ret && strcmp(ret, "") == 0) { - free(ret); - ret = NULL; + err = nvmf_get_discovery_log(c, &log, MAX_DISC_RETRIES); + if (err) { + if (err > 0) + nvme_show_status(err); + else + fprintf(stderr, "failed to get discovery log: %s\n", + nvme_strerror(errno)); + return err; } - return ret; -} - -static char *hostnqn_generate_systemd(void) -{ -#ifdef HAVE_SYSTEMD - sd_id128_t id; - char *ret; - if (sd_id128_get_machine_app_specific(NVME_HOSTNQN_ID, &id) < 0) - return NULL; - if (asprintf(&ret, "nqn.2014-08.org.nvmexpress:uuid:" SD_ID128_FORMAT_STR "\n", SD_ID128_FORMAT_VAL(id)) == -1) - ret = NULL; - - return ret; -#else - return NULL; -#endif -} - -static char *hostnqn_read_dmi(void) -{ - char uuid[16]; - char *ret = NULL; - - if (uuid_from_dmi(uuid) < 0) - return NULL; - if (asprintf(&ret, "nqn.2014-08.org.nvmexpress:uuid:%s", uuid) == -1) - return NULL; - return ret; -} - -/* returns an allocated string or NULL */ -char *hostnqn_read(void) -{ - char *ret; - - ret = hostnqn_read_file(); - if (ret) - return ret; - - ret = hostnqn_read_dmi(); - if (ret) - return ret; - - ret = hostnqn_generate_systemd(); - if (ret) - return ret; - - return NULL; -} - -static int nvmf_hostnqn_file(void) -{ - fabrics_cfg.hostnqn = hostnqn_read(); - - return fabrics_cfg.hostnqn != NULL; -} - -static int nvmf_hostid_file(void) -{ - FILE *f; - char hostid[NVMF_HOSTID_SIZE + 1]; - int ret = false; - - f = fopen(PATH_NVMF_HOSTID, "r"); - if (f == NULL) - return false; - - if (fgets(hostid, sizeof(hostid), f) == NULL) - goto out; - - fabrics_cfg.hostid = strdup(hostid); - if (!fabrics_cfg.hostid) - goto out; - - ret = true; -out: - fclose(f); - return ret; -} - -static int -add_bool_argument(char **argstr, int *max_len, char *arg_str, bool arg) -{ - int len; - - if (arg) { - len = snprintf(*argstr, *max_len, ",%s", arg_str); - if (len < 0) - return -EINVAL; - *argstr += len; - *max_len -= len; + numrec = le64_to_cpu(log->numrec); + if (raw) + save_discovery_log(raw, log); + else if (!connect) { + if (flags == JSON) + json_discovery_log(log, numrec); + else + print_discovery_log(log, numrec); + } else if (connect) { + int i; + + for (i = 0; i < numrec; i++) { + struct nvmf_disc_log_entry *e = &log->entries[i]; + bool discover = false; + nvme_ctrl_t child; + int tmo = defcfg->keep_alive_tmo; + + if (e->subtype == NVME_NQN_DISC) + set_discovery_kato(defcfg); + + errno = 0; + child = nvmf_connect_disc_entry(h, e, defcfg, + &discover); + + defcfg->keep_alive_tmo = tmo; + + if (child) { + if (discover) + __discover(child, defcfg, raw, + true, persistent, flags); + if (e->subtype != NVME_NQN_NVME && + !persistent) { + nvme_disconnect_ctrl(child); + nvme_free_ctrl(child); + } + } else if (errno == EALREADY && !quiet) { + char *traddr = log->entries[i].traddr; + + space_strip_len(NVMF_TRADDR_SIZE, traddr); + fprintf(stderr, + "traddr=%s is already connected\n", + traddr); + } + } } + free(log); return 0; } -static int -add_int_argument(char **argstr, int *max_len, char *arg_str, int arg, - bool allow_zero) +/* + * Compare two C strings and handle NULL pointers gracefully. + * If either of the two strings is NULL, return 0 + * to let caller ignore the compare. + */ +static inline int strcmp0(const char *s1, const char *s2) { - int len; - - if (arg || allow_zero) { - len = snprintf(*argstr, *max_len, ",%s=%d", arg_str, arg); - if (len < 0) - return -EINVAL; - *argstr += len; - *max_len -= len; - } - - return 0; + if (!s1 || !s2) + return 0; + return strcmp(s1, s2); } -static int -add_argument(char **argstr, int *max_len, char *arg_str, const char *arg) +/* + * Compare two C strings and handle NULL pointers gracefully. + * If either of the two strings is NULL, return 0 + * to let caller ignore the compare. + */ +static inline int strcasecmp0(const char *s1, const char *s2) { - int len; - - if (arg && strcmp(arg, "none")) { - len = snprintf(*argstr, *max_len, ",%s=%s", arg_str, arg); - if (len < 0) - return -EINVAL; - *argstr += len; - *max_len -= len; - } - - return 0; + if (!s1 || !s2) + return 0; + return strcasecmp(s1, s2); } -int build_options(char *argstr, int max_len, bool discover) +static bool ctrl_config_match(nvme_ctrl_t c, struct tr_config *trcfg) { - int len; + if (!strcmp0(nvme_ctrl_get_transport(c), trcfg->transport) && + !strcasecmp0(nvme_ctrl_get_traddr(c), trcfg->traddr) && + !strcmp0(nvme_ctrl_get_trsvcid(c), trcfg->trsvcid) && + !strcmp0(nvme_ctrl_get_host_traddr(c), trcfg->host_traddr) && + !strcmp0(nvme_ctrl_get_host_iface(c), trcfg->host_iface)) + return true; - if (!fabrics_cfg.transport) { - msg(LOG_ERR, "need a transport (-t) argument\n"); - return -EINVAL; - } + return false; +} - if (strncmp(fabrics_cfg.transport, "loop", 4)) { - if (!fabrics_cfg.traddr) { - msg(LOG_ERR, "need a address (-a) argument\n"); - return -EINVAL; +static nvme_ctrl_t lookup_discover_ctrl(nvme_root_t r, struct tr_config *trcfg) +{ + nvme_host_t h; + nvme_subsystem_t s; + nvme_ctrl_t c; + + nvme_for_each_host(r, h) { + nvme_for_each_subsystem(h, s) { + nvme_subsystem_for_each_ctrl(s, c) { + if (!nvme_ctrl_is_discovery_ctrl(c)) + continue; + if (ctrl_config_match(c, trcfg)) + return c; + } } } - /* always specify nqn as first arg - this will init the string */ - len = snprintf(argstr, max_len, "nqn=%s", fabrics_cfg.nqn); - if (len < 0) - return -EINVAL; - argstr += len; - max_len -= len; - - if (add_argument(&argstr, &max_len, "transport", fabrics_cfg.transport) || - add_argument(&argstr, &max_len, "traddr", fabrics_cfg.traddr) || - add_argument(&argstr, &max_len, "host_traddr", fabrics_cfg.host_traddr) || - add_argument(&argstr, &max_len, "host_iface", fabrics_cfg.host_iface) || - add_argument(&argstr, &max_len, "trsvcid", fabrics_cfg.trsvcid) || - ((fabrics_cfg.hostnqn || nvmf_hostnqn_file()) && - add_argument(&argstr, &max_len, "hostnqn", fabrics_cfg.hostnqn)) || - ((fabrics_cfg.hostid || nvmf_hostid_file()) && - add_argument(&argstr, &max_len, "hostid", fabrics_cfg.hostid)) || - (!discover && - add_int_argument(&argstr, &max_len, "nr_io_queues", - fabrics_cfg.nr_io_queues, false)) || - add_int_argument(&argstr, &max_len, "nr_write_queues", - fabrics_cfg.nr_write_queues, false) || - add_int_argument(&argstr, &max_len, "nr_poll_queues", - fabrics_cfg.nr_poll_queues, false) || - (!discover && - add_int_argument(&argstr, &max_len, "queue_size", - fabrics_cfg.queue_size, false)) || - add_int_argument(&argstr, &max_len, "keep_alive_tmo", - fabrics_cfg.keep_alive_tmo, false) || - add_int_argument(&argstr, &max_len, "reconnect_delay", - fabrics_cfg.reconnect_delay, false) || - (strncmp(fabrics_cfg.transport, "loop", 4) && - add_int_argument(&argstr, &max_len, "ctrl_loss_tmo", - fabrics_cfg.ctrl_loss_tmo, true)) || - (fabrics_cfg.fast_io_fail_tmo != -1 && - add_int_argument(&argstr, &max_len, "fast_io_fail_tmo", - fabrics_cfg.fast_io_fail_tmo, true)) || - (fabrics_cfg.tos != -1 && - add_int_argument(&argstr, &max_len, "tos", - fabrics_cfg.tos, true)) || - add_bool_argument(&argstr, &max_len, "duplicate_connect", - fabrics_cfg.duplicate_connect) || - add_bool_argument(&argstr, &max_len, "disable_sqflow", - fabrics_cfg.disable_sqflow) || - add_bool_argument(&argstr, &max_len, "hdr_digest", fabrics_cfg.hdr_digest) || - add_bool_argument(&argstr, &max_len, "data_digest", fabrics_cfg.data_digest)) - return -EINVAL; - - return 0; -} - -static void set_discovery_kato(struct fabrics_config *cfg) -{ - /* Set kato to NVMF_DEF_DISC_TMO for persistent controllers */ - if (cfg->persistent && !cfg->keep_alive_tmo) - cfg->keep_alive_tmo = NVMF_DEF_DISC_TMO; - /* Set kato to zero for non-persistent controllers */ - else if (!cfg->persistent && (cfg->keep_alive_tmo > 0)) - cfg->keep_alive_tmo = 0; + return NULL; } -static void discovery_trsvcid(struct fabrics_config *fabrics_cfg, bool discover) +static char *get_default_trsvcid(const char *transport, + bool discovery_ctrl) { - if (!strcmp(fabrics_cfg->transport, "tcp")) { - if (discover) { + if (!strcmp(transport, "tcp")) { + if (discovery_ctrl) { /* Default port for NVMe/TCP discovery controllers */ - fabrics_cfg->trsvcid = __stringify(NVME_DISC_IP_PORT); + return stringify(NVME_DISC_IP_PORT); } else { /* Default port for NVMe/TCP io controllers */ - fabrics_cfg->trsvcid = __stringify(NVME_RDMA_IP_PORT); + return stringify(NVME_RDMA_IP_PORT); } - } else if (!strcmp(fabrics_cfg->transport, "rdma")) { + } else if (!strcmp(transport, "rdma")) { /* Default port for NVMe/RDMA controllers */ - fabrics_cfg->trsvcid = __stringify(NVME_RDMA_IP_PORT); + return stringify(NVME_RDMA_IP_PORT); } -} -static bool traddr_is_hostname(struct fabrics_config *fabrics_cfg) -{ - char addrstr[NVMF_TRADDR_SIZE]; - - if (!fabrics_cfg->traddr || !fabrics_cfg->transport) - return false; - if (strcmp(fabrics_cfg->transport, "tcp") && strcmp(fabrics_cfg->transport, "rdma")) - return false; - if (inet_pton(AF_INET, fabrics_cfg->traddr, addrstr) > 0 || - inet_pton(AF_INET6, fabrics_cfg->traddr, addrstr) > 0) - return false; - return true; + return NULL; } -static int hostname2traddr(struct fabrics_config *fabrics_cfg) +static int discover_from_conf_file(nvme_root_t r, nvme_host_t h, + const char *desc, bool connect, + const struct nvme_fabrics_config *defcfg) { - struct addrinfo *host_info, hints = {.ai_family = AF_UNSPEC}; - char addrstr[NVMF_TRADDR_SIZE]; - const char *p; - int ret; - - ret = getaddrinfo(fabrics_cfg->traddr, NULL, &hints, &host_info); - if (ret) { - msg(LOG_ERR, "failed to resolve host %s info\n", fabrics_cfg->traddr); - return ret; - } - - switch (host_info->ai_family) { - case AF_INET: - p = inet_ntop(host_info->ai_family, - &(((struct sockaddr_in *)host_info->ai_addr)->sin_addr), - addrstr, NVMF_TRADDR_SIZE); - break; - case AF_INET6: - p = inet_ntop(host_info->ai_family, - &(((struct sockaddr_in6 *)host_info->ai_addr)->sin6_addr), - addrstr, NVMF_TRADDR_SIZE); - break; - default: - msg(LOG_ERR, "unrecognized address family (%d) %s\n", - host_info->ai_family, fabrics_cfg->traddr); - ret = -EINVAL; - goto free_addrinfo; - } - - if (!p) { - msg(LOG_ERR, "failed to get traddr for %s\n", fabrics_cfg->traddr); - ret = -errno; - goto free_addrinfo; - } - fabrics_cfg->traddr = strdup(addrstr); - -free_addrinfo: - freeaddrinfo(host_info); - return ret; -} + char *transport = NULL, *traddr = NULL, *trsvcid = NULL; + char *hostnqn = NULL, *hostid = NULL, *hostkey = NULL, *ctrlkey = NULL; + char *subsysnqn = NULL; + char *ptr, **argv, *p, line[4096]; + int argc, ret = 0; + unsigned int verbose = 0; + FILE *f; + enum nvme_print_flags flags; + char *format = "normal"; + struct nvme_fabrics_config cfg; + bool force = false; -static int connect_ctrl(struct nvmf_disc_rsp_page_entry *e) -{ - char argstr[BUF_SIZE], *p; - const char *transport; - bool discover, disable_sqflow = true; - int len, ret; - -retry: - p = argstr; - discover = false; - - switch (e->subtype) { - case NVME_NQN_DISC: - discover = true; - case NVME_NQN_NVME: - break; - default: - msg(LOG_ERR, "skipping unsupported subtype %d\n", - e->subtype); - return -EINVAL; - } + OPT_ARGS(opts) = { + NVMF_OPTS(cfg), + OPT_FMT("output-format", 'o', &format, output_format), + OPT_FILE("raw", 'r', &raw, "save raw output to file"), + OPT_FLAG("persistent", 'p', &persistent, "persistent discovery connection"), + OPT_FLAG("quiet", 'S', &quiet, "suppress already connected errors"), + OPT_INCR("verbose", 'v', &verbose, "Increase logging verbosity"), + OPT_FLAG("force", 0, &force, "Force persistent discovery controller creation"), + OPT_END() + }; - len = sprintf(p, "nqn=%s", e->subnqn); - if (len < 0) - return -EINVAL; - p += len; + nvmf_default_config(&cfg); - if (fabrics_cfg.hostnqn && strcmp(fabrics_cfg.hostnqn, "none")) { - len = sprintf(p, ",hostnqn=%s", fabrics_cfg.hostnqn); - if (len < 0) - return -EINVAL; - p += len; - } + ret = flags = validate_output_format(format); + if (ret < 0) + return ret; - if (fabrics_cfg.hostid && strcmp(fabrics_cfg.hostid, "none")) { - len = sprintf(p, ",hostid=%s", fabrics_cfg.hostid); - if (len < 0) - return -EINVAL; - p += len; + f = fopen(PATH_NVMF_DISC, "r"); + if (f == NULL) { + errno = ENOENT; + return -1; } - if (fabrics_cfg.queue_size && !discover) { - len = sprintf(p, ",queue_size=%d", fabrics_cfg.queue_size); - if (len < 0) - return -EINVAL; - p += len; + argv = calloc(MAX_DISC_ARGS, sizeof(char *)); + if (!argv) { + ret = -1; + goto out; } - if (fabrics_cfg.nr_io_queues && !discover) { - len = sprintf(p, ",nr_io_queues=%d", fabrics_cfg.nr_io_queues); - if (len < 0) - return -EINVAL; - p += len; - } + argv[0] = "discover"; + memset(line, 0, sizeof(line)); + while (fgets(line, sizeof(line), f) != NULL) { + nvme_ctrl_t c; - if (fabrics_cfg.nr_write_queues) { - len = sprintf(p, ",nr_write_queues=%d", fabrics_cfg.nr_write_queues); - if (len < 0) - return -EINVAL; - p += len; - } + if (line[0] == '#' || line[0] == '\n') + continue; - if (fabrics_cfg.nr_poll_queues) { - len = sprintf(p, ",nr_poll_queues=%d", fabrics_cfg.nr_poll_queues); - if (len < 0) - return -EINVAL; - p += len; - } + argc = 1; + p = line; + while ((ptr = strsep(&p, " =\n")) != NULL) + argv[argc++] = ptr; + argv[argc] = NULL; - if (fabrics_cfg.host_traddr && strcmp(fabrics_cfg.host_traddr, "none")) { - len = sprintf(p, ",host_traddr=%s", fabrics_cfg.host_traddr); - if (len < 0) - return -EINVAL; - p+= len; - } + memcpy(&cfg, defcfg, sizeof(cfg)); + subsysnqn = NVME_DISC_SUBSYS_NAME; + ret = argconfig_parse(argc, argv, desc, opts); + if (ret) + goto next; + if (!transport && !traddr) + goto next; + + if (!trsvcid) + trsvcid = get_default_trsvcid(transport, true); + + struct tr_config trcfg = { + .subsysnqn = subsysnqn, + .transport = transport, + .traddr = traddr, + .host_traddr = cfg.host_traddr, + .host_iface = cfg.host_iface, + .trsvcid = trsvcid, + }; + + if (!force) { + c = lookup_discover_ctrl(r, &trcfg); + if (c) { + __discover(c, &cfg, raw, connect, + true, flags); + goto next; + } + } - if (fabrics_cfg.host_iface && strcmp(fabrics_cfg.host_iface, "none")) { - len = sprintf(p, ",host_iface=%s", fabrics_cfg.host_iface); - if (len < 0) - return -EINVAL; - p+= len; - } + c = create_discover_ctrl(r, h, &cfg, &trcfg); + if (!c) + goto next; - if (fabrics_cfg.reconnect_delay) { - len = sprintf(p, ",reconnect_delay=%d", fabrics_cfg.reconnect_delay); - if (len < 0) - return -EINVAL; - p += len; - } + __discover(c, &cfg, raw, connect, persistent, flags); + if (!persistent) + ret = nvme_disconnect_ctrl(c); + nvme_free_ctrl(c); - if ((e->trtype != NVMF_TRTYPE_LOOP) && (fabrics_cfg.ctrl_loss_tmo >= -1)) { - len = sprintf(p, ",ctrl_loss_tmo=%d", fabrics_cfg.ctrl_loss_tmo); - if (len < 0) - return -EINVAL; - p += len; +next: + memset(&cfg, 0, sizeof(cfg)); } + free(argv); +out: + fclose(f); + return ret; +} - if (fabrics_cfg.fast_io_fail_tmo) { - len = sprintf(p, ",fast_io_fail_tmo=%d", fabrics_cfg.fast_io_fail_tmo); - if (len < 0) - return -EINVAL; - p += len; - } +int nvmf_discover(const char *desc, int argc, char **argv, bool connect) +{ + char *subsysnqn = NVME_DISC_SUBSYS_NAME; + char *hostnqn = NULL, *hostid = NULL, *hostkey = NULL, *ctrlkey = NULL; + char *transport = NULL, *traddr = NULL, *trsvcid = NULL; + char *config_file = PATH_NVMF_CONFIG; + char *hnqn = NULL, *hid = NULL; + enum nvme_print_flags flags; + nvme_root_t r; + nvme_host_t h; + nvme_ctrl_t c = NULL; + unsigned int verbose = 0; + int ret; + char *format = "normal"; + struct nvme_fabrics_config cfg; + char *device = NULL; + bool force = false; - if (fabrics_cfg.tos != -1) { - len = sprintf(p, ",tos=%d", fabrics_cfg.tos); - if (len < 0) - return -EINVAL; - p += len; - } + OPT_ARGS(opts) = { + OPT_STRING("device", 'd', "DEV", &device, "use existing discovery controller device"), + NVMF_OPTS(cfg), + OPT_FMT("output-format", 'o', &format, output_format), + OPT_FILE("raw", 'r', &raw, "save raw output to file"), + OPT_FLAG("persistent", 'p', &persistent, "persistent discovery connection"), + OPT_FLAG("quiet", 'S', &quiet, "suppress already connected errors"), + OPT_STRING("config", 'J', "FILE", &config_file, nvmf_config_file), + OPT_INCR("verbose", 'v', &verbose, "Increase logging verbosity"), + OPT_FLAG("dump-config", 'O', &dump_config, "Dump configuration file to stdout"), + OPT_FLAG("force", 0, &force, "Force persistent discovery controller creation"), + OPT_END() + }; - if (fabrics_cfg.keep_alive_tmo) { - len = sprintf(p, ",keep_alive_tmo=%d", fabrics_cfg.keep_alive_tmo); - if (len < 0) - return -EINVAL; - p += len; - } + nvmf_default_config(&cfg); - transport = trtype_str(e->trtype); - if (!strcmp(transport, "unrecognized")) { - msg(LOG_ERR, "skipping unsupported transport %d\n", - e->trtype); - return -EINVAL; - } + ret = argconfig_parse(argc, argv, desc, opts); + if (ret) + return ret; - len = sprintf(p, ",transport=%s", transport); - if (len < 0) - return -EINVAL; - p += len; + ret = flags = validate_output_format(format); + if (ret < 0) + return ret; - if (fabrics_cfg.hdr_digest) { - len = sprintf(p, ",hdr_digest"); - if (len < 0) - return -EINVAL; - p += len; - } + if (!strcmp(config_file, "none")) + config_file = NULL; - if (fabrics_cfg.data_digest) { - len = sprintf(p, ",data_digest"); - if (len < 0) - return -EINVAL; - p += len; + r = nvme_create_root(stderr, map_log_level(verbose, quiet)); + if (!r) { + fprintf(stderr, "Failed to create topology root: %s\n", + nvme_strerror(errno)); + return -errno; } - - switch (e->trtype) { - case NVMF_TRTYPE_RDMA: - case NVMF_TRTYPE_TCP: - switch (e->adrfam) { - case NVMF_ADDR_FAMILY_IP4: - case NVMF_ADDR_FAMILY_IP6: - /* FALLTHRU */ - len = sprintf(p, ",traddr=%.*s", - space_strip_len(NVMF_TRADDR_SIZE, e->traddr), - e->traddr); - if (len < 0) - return -EINVAL; - p += len; - - len = sprintf(p, ",trsvcid=%.*s", - space_strip_len(NVMF_TRSVCID_SIZE, e->trsvcid), - e->trsvcid); - if (len < 0) - return -EINVAL; - p += len; - break; - default: - msg(LOG_ERR, "skipping unsupported adrfam\n"); - return -EINVAL; - } - break; - case NVMF_TRTYPE_FC: - switch (e->adrfam) { - case NVMF_ADDR_FAMILY_FC: - len = sprintf(p, ",traddr=%.*s", - space_strip_len(NVMF_TRADDR_SIZE, e->traddr), - e->traddr); - if (len < 0) - return -EINVAL; - p += len; - break; - default: - msg(LOG_ERR, "skipping unsupported adrfam\n"); - return -EINVAL; - } - break; + ret = nvme_scan_topology(r, NULL, NULL); + if (ret < 0) { + fprintf(stderr, "Failed to scan topoplogy: %s\n", + nvme_strerror(errno)); + nvme_free_tree(r); + return ret; } - if (e->treq & NVMF_TREQ_DISABLE_SQFLOW && disable_sqflow) { - len = sprintf(p, ",disable_sqflow"); - if (len < 0) - return -EINVAL; - p += len; - } + if (!hostnqn) + hostnqn = hnqn = nvmf_hostnqn_from_file(); + if (!hostnqn) + hostnqn = hnqn = nvmf_hostnqn_generate(); + if (!hostid) + hostid = hid = nvmf_hostid_from_file(); + h = nvme_lookup_host(r, hostnqn, hostid); + if (!h) { + ret = ENOMEM; + goto out_free; + } + if (device) { + if (!strcmp(device, "none")) + device = NULL; + else if (!strncmp(device, "/dev/", 5)) + device += 5; + } + if (hostkey) + nvme_host_set_dhchap_key(h, hostkey); + + if (!device && !transport && !traddr) { + ret = discover_from_conf_file(r, h, desc, connect, &cfg); + goto out_free; + } + + if (!trsvcid) + trsvcid = get_default_trsvcid(transport, true); + + struct tr_config trcfg = { + .subsysnqn = subsysnqn, + .transport = transport, + .traddr = traddr, + .host_traddr = cfg.host_traddr, + .host_iface = cfg.host_iface, + .trsvcid = trsvcid, + }; - if (discover) { - enum nvme_print_flags flags; - - flags = validate_output_format(fabrics_cfg.output_format); - if (flags < 0) - flags = NORMAL; - ret = do_discover(argstr, true, flags); - } else - ret = add_ctrl(argstr); - if (ret == -EINVAL && disable_sqflow && - e->treq & NVMF_TREQ_DISABLE_SQFLOW) { - /* disable_sqflow param might not be supported, try without it */ - disable_sqflow = false; - goto retry; + if (device && !force) { + c = nvme_scan_ctrl(r, device); + if (c) { + /* Check if device matches command-line options */ + if (!ctrl_config_match(c, &trcfg)) { + fprintf(stderr, + "ctrl device %s found, ignoring " + "non matching command-line options\n", + device); + } + + if (!nvme_ctrl_is_discovery_ctrl(c)) { + fprintf(stderr, + "ctrl device %s found, ignoring " + "non discovery controller\n", + device); + + nvme_free_ctrl(c); + c = NULL; + persistent = false; + } else { + /* + * If the controller device is found it must + * be persistent, and shouldn't be disconnected + * on exit. + */ + persistent = true; + } + } else { + /* + * No controller found, fall back to create one. + * But that controller cannot be persistent. + */ + fprintf(stderr, + "ctrl device %s not found%s\n", device, + persistent ? ", ignoring --persistent" : ""); + persistent = false; + } } - return ret; -} - -static bool cargs_match_found(struct nvmf_disc_rsp_page_entry *entry) -{ - struct connect_args cargs __cleanup__(destruct_connect_args) = { NULL, }; - struct connect_args *c = tracked_ctrls; - - cargs.traddr = strdup(entry->traddr); - cargs.transport = strdup(trtype_str(entry->trtype)); - cargs.subsysnqn = strdup(entry->subnqn); - cargs.trsvcid = strdup(entry->trsvcid); - cargs.host_traddr = strdup(fabrics_cfg.host_traddr ?: "\0"); - cargs.host_iface = strdup(fabrics_cfg.host_iface ?: "\0"); - - /* check if we have a match in the discovery recursion */ - while (c) { - if (!strcmp(cargs.subsysnqn, c->subsysnqn) && - !strcmp(cargs.transport, c->transport) && - !strcmp(cargs.traddr, c->traddr) && - !strcmp(cargs.trsvcid, c->trsvcid) && - !strcmp(cargs.host_traddr, c->host_traddr) && - !strcmp(cargs.host_iface, c->host_iface)) - return true; - c = c->next; + if (!c && !force) { + c = lookup_discover_ctrl(r, &trcfg); + if (c) + persistent = true; + } + if (!c) { + /* No device or non-matching device, create a new controller */ + c = create_discover_ctrl(r, h, &cfg, &trcfg); + if (!c) { + fprintf(stderr, + "failed to add controller, error %s\n", + nvme_strerror(errno)); + ret = errno; + goto out_free; + } } - /* check if we have a matching existing controller */ - return find_ctrl_with_connectargs(&cargs) != NULL; -} - -static bool should_connect(struct nvmf_disc_rsp_page_entry *entry) -{ - int len; - - if (cargs_match_found(entry)) - return false; + ret = __discover(c, &cfg, raw, connect, + persistent, flags); + if (!persistent) + nvme_disconnect_ctrl(c); + nvme_free_ctrl(c); - /* skip connect if the transport type doesn't match */ - if (strcmp(fabrics_cfg.transport, trtype_str(entry->trtype))) - return false; +out_free: + free(hnqn); + free(hid); + if (dump_config) + nvme_dump_config(r); + nvme_free_tree(r); - if (!fabrics_cfg.matching_only || !fabrics_cfg.traddr) - return true; - - len = space_strip_len(NVMF_TRADDR_SIZE, entry->traddr); - return !strncmp(fabrics_cfg.traddr, entry->traddr, len); + return ret; } -static int connect_ctrls(struct nvmf_disc_rsp_page_hdr *log, int numrec) +int nvmf_connect(const char *desc, int argc, char **argv) { - int i; - int instance; - int ret = 0; - - for (i = 0; i < numrec; i++) { - if (!should_connect(&log->entries[i])) - continue; + char *subsysnqn = NULL; + char *transport = NULL, *traddr = NULL; + char *trsvcid = NULL, *hostnqn = NULL, *hostid = NULL; + char *hostkey = NULL, *ctrlkey = NULL; + char *hnqn = NULL, *hid = NULL; + char *config_file = PATH_NVMF_CONFIG; + unsigned int verbose = 0; + nvme_root_t r; + nvme_host_t h; + nvme_ctrl_t c; + int ret; + struct nvme_fabrics_config cfg; + enum nvme_print_flags flags = -1; + char *format = ""; - instance = connect_ctrl(&log->entries[i]); + OPT_ARGS(opts) = { + NVMF_OPTS(cfg), + OPT_STRING("config", 'J', "FILE", &config_file, nvmf_config_file), + OPT_INCR("verbose", 'v', &verbose, "Increase logging verbosity"), + OPT_FLAG("dump-config", 'O', &dump_config, "Dump JSON configuration to stdout"), + OPT_FMT("output-format", 'o', &format, "Output format: normal|json"), + OPT_END() + }; - /* clean success */ - if (instance >= 0) - continue; + nvmf_default_config(&cfg); - /* already connected print message */ - if (instance == -EALREADY) { - const char *traddr = log->entries[i].traddr; + ret = argconfig_parse(argc, argv, desc, opts); + if (ret) + return ret; - msg(LOG_NOTICE, "traddr=%.*s is already connected\n", - space_strip_len(NVMF_TRADDR_SIZE, - traddr), - traddr); - continue; - } + if (!strcmp(format, "")) + flags = -1; + else if (!strcmp(format, "normal")) + flags = NORMAL; + else if (!strcmp(format, "json")) + flags = JSON; + else + return EINVAL; - /* - * don't error out. The Discovery Log may contain - * devices that aren't necessarily connectable via - * the system/host transport port. Let those items - * fail and continue on to the next log element. - */ + if (!subsysnqn) { + fprintf(stderr, + "required argument [--nqn | -n] not specified\n"); + return EINVAL; } - return ret; -} - -static void nvmf_get_host_identifiers(int ctrl_instance) -{ - char *path; - - if (asprintf(&path, "%s/nvme%d", SYS_NVME, ctrl_instance) < 0) - return; - fabrics_cfg.hostnqn = nvme_get_ctrl_attr(path, "hostnqn"); - fabrics_cfg.hostid = nvme_get_ctrl_attr(path, "hostid"); -} - -static DEFINE_CLEANUP_FUNC(cleanup_log, struct nvmf_disc_rsp_page_hdr *, free); - -int do_discover(char *argstr, bool connect, enum nvme_print_flags flags) -{ - struct nvmf_disc_rsp_page_hdr *log __cleanup__(cleanup_log) = NULL; - char *dev_name; - int instance, numrec = 0, ret, err; - int status = 0; - struct connect_args *cargs; - - cargs = extract_connect_args(argstr); - if (!cargs) - return -ENOMEM; - - if (fabrics_cfg.device && - !ctrl_matches_connectargs(fabrics_cfg.device, cargs)) { - free(fabrics_cfg.device); - fabrics_cfg.device = NULL; + if (!transport) { + fprintf(stderr, + "required argument [--transport | -t] not specified\n"); + return EINVAL; } - if (!fabrics_cfg.device) - fabrics_cfg.device = find_ctrl_with_connectargs(cargs); - free_connect_args(cargs); - if (!fabrics_cfg.device) { - instance = add_ctrl(argstr); - } else { - instance = ctrl_instance(fabrics_cfg.device); - nvmf_get_host_identifiers(instance); + if (strcmp(transport, "loop")) { + if (!traddr) { + fprintf(stderr, + "required argument [--address | -a] not specified for transport %s\n", + transport); + return EINVAL; + } } - if (instance < 0) - return instance; - if (asprintf(&dev_name, "/dev/nvme%d", instance) < 0) + if (!strcmp(config_file, "none")) + config_file = NULL; + + r = nvme_create_root(stderr, map_log_level(verbose, quiet)); + if (!r) { + fprintf(stderr, "Failed to create topology root: %s\n", + nvme_strerror(errno)); return -errno; - ret = nvmf_get_log_page_discovery(dev_name, &log, &numrec, &status); - free(dev_name); - if (!fabrics_cfg.device && !fabrics_cfg.persistent) { - err = remove_ctrl(instance); - if (err) - return err; } - - switch (ret) { - case DISC_OK: - if (connect) - ret = connect_ctrls(log, numrec); - else if (fabrics_cfg.raw || flags == BINARY) - save_discovery_log(log, numrec); + ret = nvme_scan_topology(r, NULL, NULL); + if (ret < 0) { + fprintf(stderr, "Failed to scan topoplogy: %s\n", + nvme_strerror(errno)); + nvme_free_tree(r); + return ret; + } + nvme_read_config(r, config_file); + + if (!hostnqn) + hostnqn = hnqn = nvmf_hostnqn_from_file(); + if (!hostnqn) + hostnqn = hnqn = nvmf_hostnqn_generate(); + if (!hostid) + hostid = hid = nvmf_hostid_from_file(); + h = nvme_lookup_host(r, hostnqn, hostid); + if (!h) { + errno = ENOMEM; + goto out_free; + } + if (hostkey) + nvme_host_set_dhchap_key(h, hostkey); + if (!trsvcid) + trsvcid = get_default_trsvcid(transport, false); + c = nvme_create_ctrl(r, subsysnqn, transport, traddr, + cfg.host_traddr, cfg.host_iface, trsvcid); + if (!c) { + errno = ENOMEM; + goto out_free; + } + if (ctrlkey) + nvme_ctrl_set_dhchap_key(c, ctrlkey); + + errno = 0; + ret = nvmf_add_ctrl(h, c, &cfg); + if (ret) + fprintf(stderr, "no controller found: %s\n", + nvme_strerror(errno)); + else { + errno = 0; + if (flags == NORMAL) + print_connect_msg(c); else if (flags == JSON) - json_discovery_log(log, numrec, instance); - else - print_discovery_log(log, numrec, instance); - break; - case DISC_GET_NUMRECS: - msg(LOG_ERR, - "Get number of discovery log entries failed.\n"); - ret = status; - break; - case DISC_GET_LOG: - msg(LOG_ERR, "Get discovery log entries failed.\n"); - ret = status; - break; - case DISC_NO_LOG: - fprintf(stdout, "No discovery log entries to fetch.\n"); - ret = DISC_OK; - break; - case DISC_RETRY_EXHAUSTED: - fprintf(stdout, "Discovery retries exhausted.\n"); - ret = -EAGAIN; - break; - case DISC_NOT_EQUAL: - msg(LOG_ERR, - "Numrec values of last two get discovery log page not equal\n"); - ret = -EBADSLT; - break; - default: - msg(LOG_ERR, "Get discovery log page failed: %d\n", ret); - break; + json_connect_msg(c); } - return ret; +out_free: + free(hnqn); + free(hid); + if (dump_config) + nvme_dump_config(r); + nvme_free_tree(r); + return errno; } -static int discover_from_conf_file(const char *desc, char *argstr, - const struct argconfig_commandline_options *opts, bool connect) +int nvmf_disconnect(const char *desc, int argc, char **argv) { - FILE *f; - char line[256], *ptr, *all_args, *args, **argv; - int argc, err, ret = 0; - - f = fopen(PATH_NVMF_DISC, "r"); - if (f == NULL) { - msg(LOG_ERR, "No discover params given and no %s\n", - PATH_NVMF_DISC); - return -EINVAL; - } - - while (fgets(line, sizeof(line), f) != NULL) { - enum nvme_print_flags flags; + const char *device = "nvme device handle"; + nvme_root_t r; + nvme_host_t h; + nvme_subsystem_t s; + nvme_ctrl_t c; + char *p; + int ret; - if (line[0] == '#' || line[0] == '\n') - continue; + struct config { + char *nqn; + char *device; + unsigned int verbose; + }; - args = strdup(line); - if (!args) { - msg(LOG_ERR, "failed to strdup args\n"); - ret = -ENOMEM; - goto out; - } - all_args = args; + struct config cfg = { 0 }; - argv = calloc(MAX_DISC_ARGS, BUF_SIZE); - if (!argv) { - msg(LOG_ERR, "failed to allocate argv vector: %m\n"); - free(args); - ret = -ENOMEM; - goto out; - } + OPT_ARGS(opts) = { + OPT_STRING("nqn", 'n', "NAME", &cfg.nqn, nvmf_nqn), + OPT_STRING("device", 'd', "DEV", &cfg.device, device), + OPT_INCR("verbose", 'v', &cfg.verbose, "Increase logging verbosity"), + OPT_END() + }; - argc = 0; - argv[argc++] = "discover"; - while ((ptr = strsep(&args, " =\n")) != NULL) - argv[argc++] = ptr; + ret = argconfig_parse(argc, argv, desc, opts); + if (ret) + return ret; - err = argconfig_parse(argc, argv, desc, opts); - if (err) - goto free_and_continue; + if (!cfg.nqn && !cfg.device) { + fprintf(stderr, + "Neither device name [--device | -d] nor NQN [--nqn | -n] provided\n"); + return EINVAL; + } - if (!fabrics_cfg.transport || !fabrics_cfg.traddr) - goto free_and_continue; + r = nvme_create_root(stderr, map_log_level(cfg.verbose, false)); + if (!r) { + fprintf(stderr, "Failed to create topology root: %s\n", + nvme_strerror(errno)); + return -errno; + } + ret = nvme_scan_topology(r, NULL, NULL); + if (ret < 0) { + fprintf(stderr, "Failed to scan topoplogy: %s\n", + nvme_strerror(errno)); + nvme_free_tree(r); + return ret; + } - err = flags = validate_output_format(fabrics_cfg.output_format); - if (err < 0) - goto free_and_continue; - set_discovery_kato(&fabrics_cfg); + if (cfg.nqn) { + int i = 0; + char *n = cfg.nqn; - if (traddr_is_hostname(&fabrics_cfg)) { - ret = hostname2traddr(&fabrics_cfg); - if (ret) - goto out; + while ((p = strsep(&n, ",")) != NULL) { + if (!strlen(p)) + continue; + nvme_for_each_host(r, h) { + nvme_for_each_subsystem(h, s) { + if (strcmp(nvme_subsystem_get_nqn(s), p)) + continue; + nvme_subsystem_for_each_ctrl(s, c) { + if (!nvme_disconnect_ctrl(c)) + i++; + } + } + } } - - if (!fabrics_cfg.trsvcid) - discovery_trsvcid(&fabrics_cfg, true); - - err = build_options(argstr, BUF_SIZE, true); - if (err) { - ret = err; - goto free_and_continue; + printf("NQN:%s disconnected %d controller(s)\n", cfg.nqn, i); + } + + if (cfg.device) { + char *d; + + d = cfg.device; + while ((p = strsep(&d, ",")) != NULL) { + if (!strncmp(p, "/dev/", 5)) + p += 5; + c = nvme_scan_ctrl(r, p); + if (!c) { + fprintf(stderr, + "Did not find device %s: %s\n", + p, nvme_strerror(errno)); + nvme_free_tree(r); + return errno; + } + ret = nvme_disconnect_ctrl(c); + if (ret) + fprintf(stderr, + "Failed to disconnect %s: %s\n", + p, nvme_strerror(errno)); } - - err = do_discover(argstr, connect, flags); - if (err) - ret = err; - -free_and_continue: - free(all_args); - free(argv); - fabrics_cfg.transport = fabrics_cfg.traddr = - fabrics_cfg.trsvcid = fabrics_cfg.host_traddr = - fabrics_cfg.host_iface = NULL; } + nvme_free_tree(r); -out: - fclose(f); - return ret; + return 0; } -int fabrics_discover(const char *desc, int argc, char **argv, bool connect) +int nvmf_disconnect_all(const char *desc, int argc, char **argv) { - char argstr[BUF_SIZE]; + nvme_host_t h; + nvme_subsystem_t s; + nvme_root_t r; + nvme_ctrl_t c; int ret; - enum nvme_print_flags flags; - bool quiet = false; + + struct config { + char *transport; + unsigned int verbose; + }; + + struct config cfg = { 0 }; OPT_ARGS(opts) = { - OPT_LIST("transport", 't', &fabrics_cfg.transport, "transport type"), - OPT_LIST("traddr", 'a', &fabrics_cfg.traddr, "transport address"), - OPT_LIST("trsvcid", 's', &fabrics_cfg.trsvcid, "transport service id (e.g. IP port)"), - OPT_LIST("host-traddr", 'w', &fabrics_cfg.host_traddr, "host traddr (e.g. FC WWN's or IP source address)"), - OPT_LIST("host-iface", 'f', &fabrics_cfg.host_iface, "host transport interface (e.g. IP eth1, enp2s0)"), - OPT_LIST("hostnqn", 'q', &fabrics_cfg.hostnqn, "user-defined hostnqn (if default not used)"), - OPT_LIST("hostid", 'I', &fabrics_cfg.hostid, "user-defined hostid (if default not used)"), - OPT_LIST("raw", 'r', &fabrics_cfg.raw, "raw output file"), - OPT_LIST("device", 'd', &fabrics_cfg.device, "existing discovery controller device"), - OPT_INT("keep-alive-tmo", 'k', &fabrics_cfg.keep_alive_tmo, "keep alive timeout period in seconds"), - OPT_INT("reconnect-delay", 'c', &fabrics_cfg.reconnect_delay, "reconnect timeout period in seconds"), - OPT_INT("ctrl-loss-tmo", 'l', &fabrics_cfg.ctrl_loss_tmo, "controller loss timeout period in seconds"), - OPT_INT("fast_io_fail_tmo",'f',&fabrics_cfg.fast_io_fail_tmo, "fast I/O fail timeout (default off)"), - OPT_INT("tos", 'T', &fabrics_cfg.tos, "type of service"), - OPT_FLAG("hdr_digest", 'g', &fabrics_cfg.hdr_digest, "enable transport protocol header digest (TCP transport)"), - OPT_FLAG("data_digest", 'G', &fabrics_cfg.data_digest, "enable transport protocol data digest (TCP transport)"), - OPT_INT("nr-io-queues", 'i', &fabrics_cfg.nr_io_queues, "number of io queues to use (default is core count)"), - OPT_INT("nr-write-queues", 'W', &fabrics_cfg.nr_write_queues, "number of write queues to use (default 0)"), - OPT_INT("nr-poll-queues", 'P', &fabrics_cfg.nr_poll_queues, "number of poll queues to use (default 0)"), - OPT_INT("queue-size", 'Q', &fabrics_cfg.queue_size, "number of io queue elements to use (default 128)"), - OPT_FLAG("persistent", 'p', &fabrics_cfg.persistent, "persistent discovery connection"), - OPT_FLAG("quiet", 'S', &quiet, "suppress already connected errors"), - OPT_FLAG("matching", 'm', &fabrics_cfg.matching_only, "connect only records matching the traddr"), - OPT_FMT("output-format", 'o', &fabrics_cfg.output_format, output_format), + OPT_STRING("transport", 'r', "STR", (char *)&cfg.transport, nvmf_tport), + OPT_INCR("verbose", 'v', &cfg.verbose, "Increase logging verbosity"), OPT_END() }; - fabrics_cfg.tos = -1; ret = argconfig_parse(argc, argv, desc, opts); if (ret) - goto out; + return ret; - ret = flags = validate_output_format(fabrics_cfg.output_format); - if (ret < 0) - goto out; - if (fabrics_cfg.device && strcmp(fabrics_cfg.device, "none")) { - fabrics_cfg.device = strdup(fabrics_cfg.device); - if (!fabrics_cfg.device) { - ret = -ENOMEM; - goto out; - } + r = nvme_create_root(stderr, map_log_level(cfg.verbose, false)); + if (!r) { + fprintf(stderr, "Failed to create topology root: %s\n", + nvme_strerror(errno)); + return -errno; + } + ret = nvme_scan_topology(r, NULL, NULL); + if (ret < 0) { + fprintf(stderr, "Failed to scan topoplogy: %s\n", + nvme_strerror(errno)); + nvme_free_tree(r); + return ret; } - if (quiet) - log_level = LOG_WARNING; - - if (fabrics_cfg.device && !strcmp(fabrics_cfg.device, "none")) - fabrics_cfg.device = NULL; - - fabrics_cfg.nqn = NVME_DISC_SUBSYS_NAME; - - if (!fabrics_cfg.transport && !fabrics_cfg.traddr) { - ret = discover_from_conf_file(desc, argstr, opts, connect); - } else { - set_discovery_kato(&fabrics_cfg); - - if (traddr_is_hostname(&fabrics_cfg)) { - ret = hostname2traddr(&fabrics_cfg); - if (ret) - goto out; + nvme_for_each_host(r, h) { + nvme_for_each_subsystem(h, s) { + nvme_subsystem_for_each_ctrl(s, c) { + if (cfg.transport && + strcmp(cfg.transport, + nvme_ctrl_get_transport(c))) + continue; + else if (!strcmp(nvme_ctrl_get_transport(c), + "pcie")) + continue; + if (nvme_disconnect_ctrl(c)) + fprintf(stderr, + "failed to disconnect %s\n", + nvme_ctrl_get_name(c)); + } } - - if (!fabrics_cfg.trsvcid) - discovery_trsvcid(&fabrics_cfg, true); - - ret = build_options(argstr, BUF_SIZE, true); - if (ret) - goto out; - - ret = do_discover(argstr, connect, flags); } + nvme_free_tree(r); -out: - return nvme_status_to_errno(ret, true); + return 0; } -int fabrics_connect(const char *desc, int argc, char **argv) +int nvmf_config(const char *desc, int argc, char **argv) { - char argstr[BUF_SIZE]; - int instance, ret; - enum nvme_print_flags flags = -1; + char *subsysnqn = NULL; + char *transport = NULL, *traddr = NULL; + char *trsvcid = NULL, *hostnqn = NULL, *hostid = NULL; + char *hnqn = NULL, *hid = NULL; + char *hostkey = NULL, *ctrlkey = NULL; + char *config_file = PATH_NVMF_CONFIG; + unsigned int verbose = 0; + nvme_root_t r; + int ret; + struct nvme_fabrics_config cfg; + bool scan_tree = false, modify_config = false, update_config = false; OPT_ARGS(opts) = { - OPT_LIST("transport", 't', &fabrics_cfg.transport, "transport type"), - OPT_LIST("nqn", 'n', &fabrics_cfg.nqn, "nqn name"), - OPT_LIST("traddr", 'a', &fabrics_cfg.traddr, "transport address"), - OPT_LIST("trsvcid", 's', &fabrics_cfg.trsvcid, "transport service id (e.g. IP port)"), - OPT_LIST("host-traddr", 'w', &fabrics_cfg.host_traddr, "host traddr (e.g. FC WWN's or IP source address)"), - OPT_LIST("host-iface", 'f', &fabrics_cfg.host_iface, "host transport interface (e.g. IP eth1, enp2s0)"), - OPT_LIST("hostnqn", 'q', &fabrics_cfg.hostnqn, "user-defined hostnqn"), - OPT_LIST("hostid", 'I', &fabrics_cfg.hostid, "user-defined hostid (if default not used)"), - OPT_INT("nr-io-queues", 'i', &fabrics_cfg.nr_io_queues, "number of io queues to use (default is core count)"), - OPT_INT("nr-write-queues", 'W', &fabrics_cfg.nr_write_queues, "number of write queues to use (default 0)"), - OPT_INT("nr-poll-queues", 'P', &fabrics_cfg.nr_poll_queues, "number of poll queues to use (default 0)"), - OPT_INT("queue-size", 'Q', &fabrics_cfg.queue_size, "number of io queue elements to use (default 128)"), - OPT_INT("keep-alive-tmo", 'k', &fabrics_cfg.keep_alive_tmo, "keep alive timeout period in seconds"), - OPT_INT("reconnect-delay", 'c', &fabrics_cfg.reconnect_delay, "reconnect timeout period in seconds"), - OPT_INT("ctrl-loss-tmo", 'l', &fabrics_cfg.ctrl_loss_tmo, "controller loss timeout period in seconds"), - OPT_INT("fast_io_fail_tmo", 'f', &fabrics_cfg.fast_io_fail_tmo, "fast I/O fail timeout (default off)"), - OPT_INT("tos", 'T', &fabrics_cfg.tos, "type of service"), - OPT_FLAG("duplicate-connect", 'D', &fabrics_cfg.duplicate_connect, "allow duplicate connections between same transport host and subsystem port"), - OPT_FLAG("disable-sqflow", 'd', &fabrics_cfg.disable_sqflow, "disable controller sq flow control (default false)"), - OPT_FLAG("hdr-digest", 'g', &fabrics_cfg.hdr_digest, "enable transport protocol header digest (TCP transport)"), - OPT_FLAG("data-digest", 'G', &fabrics_cfg.data_digest, "enable transport protocol data digest (TCP transport)"), - OPT_FMT("output-format", 'o', &fabrics_cfg.output_format, "Output format: normal|json"), + NVMF_OPTS(cfg), + OPT_STRING("config", 'J', "FILE", &config_file, nvmf_config_file), + OPT_INCR("verbose", 'v', &verbose, "Increase logging verbosity"), + OPT_FLAG("scan", 'R', &scan_tree, "Scan current NVMeoF topology"), + OPT_FLAG("modify", 'M', &modify_config, "Modify JSON configuration file"), + OPT_FLAG("dump", 'O', &dump_config, "Dump JSON configuration to stdout"), + OPT_FLAG("update", 'U', &update_config, "Update JSON configuration file"), OPT_END() }; - fabrics_cfg.output_format = ""; - fabrics_cfg.tos = -1; + nvmf_default_config(&cfg); + ret = argconfig_parse(argc, argv, desc, opts); if (ret) - goto out; + return ret; - if (!strcmp(fabrics_cfg.output_format, "")) - flags = -1; - else if (!strcmp(fabrics_cfg.output_format, "normal")) - flags = NORMAL; - else if (!strcmp(fabrics_cfg.output_format, "json")) - flags = JSON; - else { - ret = -EINVAL; - goto out; - } + if (!strcmp(config_file, "none")) + config_file = NULL; - if (traddr_is_hostname(&fabrics_cfg)) { - ret = hostname2traddr(&fabrics_cfg); - if (ret) + r = nvme_create_root(stderr, map_log_level(verbose, quiet)); + if (!r) { + fprintf(stderr, "Failed to create topology root: %s\n", + nvme_strerror(errno)); + return -errno; + } + if (scan_tree) { + ret = nvme_scan_topology(r, NULL, NULL); + if (ret < 0) { + fprintf(stderr, "Failed to scan topoplogy: %s\n", + nvme_strerror(errno)); + nvme_free_tree(r); + return ret; + } + } + nvme_read_config(r, config_file); + + if (modify_config) { + nvme_host_t h; + nvme_subsystem_t s; + nvme_ctrl_t c; + + if (!hostnqn) + hostnqn = hnqn = nvmf_hostnqn_from_file(); + if (!hostid && hnqn) + hostid = hid = nvmf_hostid_from_file(); + h = nvme_lookup_host(r, hostnqn, hostid); + if (!h) { + fprintf(stderr, "Failed to lookup host '%s': %s\n", + hostnqn, nvme_strerror(errno)); + goto out; + } + if (hostkey) + nvme_host_set_dhchap_key(h, hostkey); + s = nvme_lookup_subsystem(h, NULL, subsysnqn); + if (!s) { + fprintf(stderr, "Failed to lookup subsystem '%s': %s\n", + subsysnqn, nvme_strerror(errno)); + goto out; + } + c = nvme_lookup_ctrl(s, transport, traddr, + cfg.host_traddr, cfg.host_iface, + trsvcid, NULL); + if (!c) { + fprintf(stderr, "Failed to lookup controller: %s\n", + nvme_strerror(errno)); goto out; + } + nvmf_update_config(c, &cfg); + if (ctrlkey) + nvme_ctrl_set_dhchap_key(c, ctrlkey); } - if (!fabrics_cfg.trsvcid) - discovery_trsvcid(&fabrics_cfg, false); - - ret = build_options(argstr, BUF_SIZE, false); - if (ret) - goto out; - - if (!fabrics_cfg.nqn) { - msg(LOG_ERR, "need a -n argument\n"); - ret = -EINVAL; - goto out; - } + if (update_config) + nvme_update_config(r); - instance = add_ctrl(argstr); - if (instance < 0) - ret = instance; - else { - if (flags == NORMAL) - print_connect_msg(instance); - else if (flags == JSON) - json_connect_msg(instance); - } + if (dump_config) + nvme_dump_config(r); out: - return nvme_status_to_errno(ret, true); -} - -static int scan_sys_nvme_filter(const struct dirent *d) -{ - if (!strcmp(d->d_name, ".")) - return 0; - if (!strcmp(d->d_name, "..")) - return 0; - return 1; + if (hid) + free(hid); + if (hnqn) + free(hnqn); + nvme_free_tree(r); + return errno; } -/* - * Returns 1 if disconnect occurred, 0 otherwise. - */ -static int disconnect_subsys(const char *nqn, char *ctrl) +static void dim_operation(nvme_ctrl_t c, enum nvmf_dim_tas tas, const char * name) { - char *sysfs_nqn_path = NULL, *sysfs_del_path = NULL; - char subsysnqn[NVMF_NQN_SIZE] = {}; - int fd, ret = 0; - - if (asprintf(&sysfs_nqn_path, "%s/%s/subsysnqn", SYS_NVME, ctrl) < 0) - goto free; - if (asprintf(&sysfs_del_path, "%s/%s/delete_controller", SYS_NVME, ctrl) < 0) - goto free; - - fd = open(sysfs_nqn_path, O_RDONLY); - if (fd < 0) { - msg(LOG_ERR, "Failed to open %s: %s\n", - sysfs_nqn_path, strerror(errno)); - goto free; + static const char * const task[] = { + [NVMF_DIM_TAS_REGISTER] = "register", + [NVMF_DIM_TAS_DEREGISTER] = "deregister", + }; + const char * t; + int status; + __u32 result; + + t = (tas > NVMF_DIM_TAS_DEREGISTER || !task[tas]) ? "reserved" : task[tas]; + status = nvmf_register_ctrl(c, tas, &result); + if (status == NVME_SC_SUCCESS) { + printf("%s DIM %s command success\n", name, t); + } else if (status < NVME_SC_SUCCESS) { + fprintf(stderr, "%s DIM %s command error. Status:0x%04x - %s\n", + name, t, status, nvme_status_to_string(status, false)); + } else { + fprintf(stderr, "%s DIM %s command error. Result:0x%04x, Status:0x%04x - %s\n", + name, t, result, status, nvme_status_to_string(status, false)); } - - if (read(fd, subsysnqn, NVMF_NQN_SIZE) < 0) - goto close; - - subsysnqn[strcspn(subsysnqn, "\n")] = '\0'; - if (strcmp(subsysnqn, nqn)) - goto close; - - if (!remove_ctrl_by_path(sysfs_del_path)) - ret = 1; - close: - close(fd); - free: - free(sysfs_del_path); - free(sysfs_nqn_path); - return ret; -} - -/* - * Returns the number of controllers successfully disconnected. - */ -static int disconnect_by_nqn(const char *nqn) -{ - struct dirent **devices = NULL; - int i, n, ret = 0; - - if (strlen(nqn) > NVMF_NQN_SIZE) - return -EINVAL; - - n = scandir(SYS_NVME, &devices, scan_sys_nvme_filter, alphasort); - if (n < 0) - return n; - - for (i = 0; i < n; i++) - ret += disconnect_subsys(nqn, devices[i]->d_name); - - for (i = 0; i < n; i++) - free(devices[i]); - free(devices); - - return ret; } -static int disconnect_by_device(const char *device) +int nvmf_dim(const char *desc, int argc, char **argv) { - int instance; - - instance = ctrl_instance(device); - if (instance < 0) - return instance; - return remove_ctrl(instance); -} - -int fabrics_disconnect(const char *desc, int argc, char **argv) -{ - const char *nqn = "nqn name"; - const char *device = "nvme device"; + enum nvmf_dim_tas tas; + nvme_root_t r; + nvme_ctrl_t c; + char *p; int ret; + struct { + char *nqn; + char *device; + char *tas; + unsigned int verbose; + } cfg = { 0 }; + OPT_ARGS(opts) = { - OPT_LIST("nqn", 'n', &fabrics_cfg.nqn, nqn), - OPT_LIST("device", 'd', &fabrics_cfg.device, device), + OPT_STRING("nqn", 'n', "NAME", &cfg.nqn, "Comma-separated list of DC nqn"), + OPT_STRING("device", 'd', "DEV", &cfg.device, "Comma-separated list of DC nvme device handle."), + OPT_STRING("task", 't', "TASK", &cfg.tas, "[register|deregister]"), + OPT_INCR("verbose", 'v', &cfg.verbose, "Increase logging verbosity"), OPT_END() }; ret = argconfig_parse(argc, argv, desc, opts); if (ret) - goto out; + return ret; - if (!fabrics_cfg.nqn && !fabrics_cfg.device) { - msg(LOG_ERR, "need a -n or -d argument\n"); - ret = -EINVAL; - goto out; + if (!cfg.nqn && !cfg.device) { + fprintf(stderr, + "Neither device name [--device | -d] nor NQN [--nqn | -n] provided\n"); + return EINVAL; } - if (fabrics_cfg.nqn) { - ret = disconnect_by_nqn(fabrics_cfg.nqn); - if (ret < 0) - msg(LOG_ERR, "Failed to disconnect by NQN: %s\n", - fabrics_cfg.nqn); - else { - printf("NQN:%s disconnected %d controller(s)\n", fabrics_cfg.nqn, ret); - ret = 0; - } + if (!cfg.tas) { + fprintf(stderr, + "Task [--task | -t] must be specified\n"); + return EINVAL; } - if (fabrics_cfg.device) { - ret = disconnect_by_device(fabrics_cfg.device); - if (ret) - msg(LOG_ERR, - "Failed to disconnect by device name: %s\n", - fabrics_cfg.device); + /* Allow partial name (e.g. "reg" for "register" */ + if (strstarts("register", cfg.tas)) { + tas = NVMF_DIM_TAS_REGISTER; + } else if (strstarts("deregister", cfg.tas)) { + tas = NVMF_DIM_TAS_DEREGISTER; + } else { + fprintf(stderr, "Invalid --task: %s\n", cfg.tas); + return EINVAL; } -out: - return nvme_status_to_errno(ret, true); -} - -int fabrics_disconnect_all(const char *desc, int argc, char **argv) -{ - struct nvme_topology t = { }; - int i, j, err; - - OPT_ARGS(opts) = { - OPT_END() - }; - - err = argconfig_parse(argc, argv, desc, opts); - if (err) - goto out; - - err = scan_subsystems(&t, NULL, 0, 0, NULL); - if (err) { - msg(LOG_ERR, "Failed to scan namespaces\n"); - goto out; + r = nvme_create_root(stderr, map_log_level(cfg.verbose, false)); + if (!r) { + fprintf(stderr, "Failed to create topology root: %s\n", + nvme_strerror(errno)); + return -errno; + } + ret = nvme_scan_topology(r, NULL, NULL); + if (ret < 0) { + fprintf(stderr, "Failed to scan topoplogy: %s\n", + nvme_strerror(errno)); + nvme_free_tree(r); + return ret; } - for (i = 0; i < t.nr_subsystems; i++) { - struct nvme_subsystem *s = &t.subsystems[i]; - - for (j = 0; j < s->nr_ctrls; j++) { - struct nvme_ctrl *c = &s->ctrls[j]; + if (cfg.nqn) { + nvme_host_t h; + nvme_subsystem_t s; + char *n = cfg.nqn; - if (!c->transport || !strcmp(c->transport, "pcie")) + while ((p = strsep(&n, ",")) != NULL) { + if (!strlen(p)) continue; - err = disconnect_by_device(c->name); - if (err) - goto free; + nvme_for_each_host(r, h) { + nvme_for_each_subsystem(h, s) { + if (strcmp(nvme_subsystem_get_nqn(s), p)) + continue; + nvme_subsystem_for_each_ctrl(s, c) { + dim_operation(c, tas, p); + } + } + } } } -free: - free_topology(&t); -out: - return nvme_status_to_errno(err, true); + + if (cfg.device) { + char *d = cfg.device; + + while ((p = strsep(&d, ",")) != NULL) { + if (!strncmp(p, "/dev/", 5)) + p += 5; + c = nvme_scan_ctrl(r, p); + if (!c) { + fprintf(stderr, + "Did not find device %s: %s\n", + p, nvme_strerror(errno)); + nvme_free_tree(r); + return errno; + } + dim_operation(c, tas, p); + } + } + + nvme_free_tree(r); + + return 0; } |