diff options
Diffstat (limited to 'src/spdk/scripts/config_converter.py')
| -rwxr-xr-x | src/spdk/scripts/config_converter.py | 727 |
1 files changed, 727 insertions, 0 deletions
diff --git a/src/spdk/scripts/config_converter.py b/src/spdk/scripts/config_converter.py new file mode 100755 index 000000000..4fa65a9d7 --- /dev/null +++ b/src/spdk/scripts/config_converter.py @@ -0,0 +1,727 @@ +#!/usr/bin/env python3 + +import configparser +import re +import sys +import json +from collections import OrderedDict + +bdev_dict = OrderedDict() +bdev_dict["bdev_set_options"] = [] +bdev_dict["bdev_split_create"] = [] +bdev_dict["bdev_nvme_set_options"] = [] +bdev_dict["bdev_nvme_attach_controller"] = [] +bdev_dict["bdev_nvme_set_hotplug"] = [] +bdev_dict["bdev_malloc_create"] = [] +bdev_dict["bdev_aio_create"] = [] +bdev_dict["bdev_pmem_create"] = [] +bdev_dict["bdev_virtio_attach_controller"] = [] + +vhost_dict = OrderedDict() +vhost_dict["vhost_create_scsi_controller"] = [] +vhost_dict["vhost_create_blk_controller"] = [] +vhost_dict["vhost_create_nvme_controller"] = [] + +iscsi_dict = OrderedDict() +iscsi_dict["iscsi_set_options"] = [] +iscsi_dict["iscsi_create_portal_group"] = [] +iscsi_dict["iscsi_create_initiator_group"] = [] +iscsi_dict["iscsi_create_target_node"] = [] + +nvmf_dict = OrderedDict() +nvmf_dict["nvmf_set_config"] = [] +nvmf_dict["nvmf_set_max_subsystems"] = [] +nvmf_dict["subsystems"] = [] + + +# dictionary with new config that will be written to new json config file +subsystem = { + "accel": None, + "interface": None, + "net_framework": None, + "bdev": bdev_dict, + "scsi": [], + "nvmf": nvmf_dict, + "nbd": [], + "vhost": vhost_dict, + "iscsi": iscsi_dict +} + + +class OptionOrderedDict(OrderedDict): + def __setitem__(self, option, value): + if option in self and isinstance(value, list): + self[option].extend(value) + return + super(OptionOrderedDict, self).__setitem__(option, value) + + +no_yes_map = {"no": False, "No": False, "Yes": True, "yes": True} + + +def generate_new_json_config(): + json_subsystem = [ + {'subsystem': "accel", 'config': None}, + {"subsystem": "interface", "config": None}, + {"subsystem": "net_framework", "config": None}, + {"subsystem": "bdev", "config": []}, + {"subsystem": "scsi", "config": None}, + {"subsystem": "nvmf", "config": []}, + {"subsystem": "nbd", "config": []}, + {"subsystem": "vhost", "config": []}, + {"subsystem": "iscsi", "config": []} + ] + for method in subsystem['bdev']: + for item in subsystem['bdev'][method]: + json_subsystem[3]['config'].append(item) + for item in subsystem['scsi']: + if json_subsystem[4]['config'] is None: + json_subsystem[4]['config'] = [] + json_subsystem[4]['config'].append(item) + for method in subsystem['nvmf']: + for item in subsystem['nvmf'][method]: + json_subsystem[5]['config'].append(item) + for method in subsystem['vhost']: + for item in subsystem['vhost'][method]: + json_subsystem[7]['config'].append(item) + for method in subsystem['iscsi']: + for item in subsystem['iscsi'][method]: + json_subsystem[8]['config'].append(item) + + return {"subsystems": json_subsystem} + + +section_to_subsystem = { + "Bdev": subsystem['bdev'], + "AIO": subsystem['bdev'], + "Malloc": subsystem['bdev'], + "Nvme": subsystem['bdev'], + "Pmem": subsystem['bdev'], + "Split": subsystem['bdev'], + "Nvmf": subsystem['nvmf'], + "Subsystem": subsystem['nvmf'], + "VhostScsi": subsystem['vhost'], + "VhostBlk": subsystem['vhost'], + "VhostNvme": subsystem['vhost'], + "VirtioUser": subsystem['bdev'], + "iSCSI": subsystem['iscsi'], + "PortalGroup": subsystem['iscsi'], + "InitiatorGroup": subsystem['iscsi'], + "TargetNode": subsystem['iscsi'] +} + + +def set_param(params, cfg_name, value): + for param in params: + if param[0] != cfg_name: + continue + if param[1] == "disable_chap": + param[3] = True if value == "None" else False + elif param[1] == "require_chap": + param[3] = True if value in ["CHAP", "Mutual"] else False + elif param[1] == "mutual_chap": + param[3] = True if value == "Mutual" else False + elif param[1] == "chap_group": + param[3] = int(value.replace("AuthGroup", "")) + elif param[2] == bool: + param[3] = True if value in ("yes", "true", "Yes") else False + elif param[2] == "hex": + param[3] = str(int(value, 16)) + elif param[2] == int: + param[3] = int(value) + elif param[2] == list: + param[3].append(value) + elif param[2] == "dev_type": + if value.lower() == "blk": + param[3] = "blk" + else: + param[3] = param[2](value.replace("\"", "")) + + +def to_json_params(params): + out = {} + for param in params: + if param[3] is not None: + out[param[1]] = param[3] + return out + + +def get_bdev_options_json(config, section): + params = [ + ["BdevIoPoolSize", "bdev_io_pool_size", int, 65536], + ["BdevIoCacheSize", "bdev_io_cache_size", int, 256] + ] + for option in config.options("Bdev"): + set_param(params, option, config.get("Bdev", option)) + + return [{"params": to_json_params(params), "method": "bdev_set_options"}] + + +def get_aio_bdev_json(config, section): + aio_json = [] + value = None + for option in config.options("AIO"): + if option == "AIO": + value = config.get("AIO", option).split("\n") + if value is None: + return aio_json + for item in value: + items = re.findall(r"\S+", item) + params = {} + params['filename'] = items[0] + params['name'] = items[1] + if len(items) == 3: + params['block_size'] = int(items[2]) + aio_json.append({ + "params": params, + "method": "bdev_aio_create" + }) + + return aio_json + + +def get_malloc_bdev_json(config, section): + malloc_json = [] + params = [ + ['NumberOfLuns', '', int, -1], + ['LunSizeInMB', '', int, 20], + ['BlockSize', '', int, 512] + ] + for option in config.options("Malloc"): + set_param(params, option, config.get("Malloc", option)) + for lun in range(0, params[0][3]): + malloc_json.append({ + "params": { + "block_size": params[2][3], + "num_blocks": params[1][3] * 1024 * 1024 / params[2][3], + "name": "Malloc%s" % lun + }, + "method": "bdev_malloc_create" + }) + + return malloc_json + + +def get_nvme_bdev_json(config, section): + params = [ + ["RetryCount", "retry_count", int, 4], + ["TimeoutuSec", "timeout_us", int, 0], + ["AdminPollRate", "nvme_adminq_poll_period_us", int, 1000000], + ["ActionOnTimeout", "action_on_timeout", str, "none"], + ["IOPollRate", "nvme_ioq_poll_period_us", int, 0], + ["HotplugEnable", "enable", bool, False], + ["AdminPollRate", "period_us", int, 1000] + ] + nvme_json = [] + for option in config.options("Nvme"): + value = config.get("Nvme", option) + if "TransportID" == option: + entry = re.findall(r"\S+", value) + nvme_name = entry[-1] + trtype = re.findall(r"trtype:\S+", value) + if trtype: + trtype = trtype[0].replace("trtype:", "").replace("\"", "") + traddr = re.findall(r"traddr:\S+", value) + if traddr: + traddr = traddr[0].replace("traddr:", "").replace("\"", "") + nvme_json.append({ + "params": { + "trtype": trtype, + "name": nvme_name, + "traddr": traddr + }, + "method": "bdev_nvme_attach_controller" + }) + else: + set_param(params, option, value) + params[3][3] = params[3][3].lower() + params[6][3] = params[6][3] * 100 + nvme_json.append({ + "params": to_json_params(params[5:7]), + "method": "bdev_nvme_set_hotplug" + }) + nvme_json.append({ + "params": to_json_params(params[0:5]), + "method": "bdev_nvme_set_options" + }) + return nvme_json + + +def get_pmem_bdev_json(config, section): + pmem_json = [] + for option in config.options(section): + if "Blk" == option: + for value in config.get(section, option).split("\n"): + items = re.findall(r"\S+", value) + pmem_json.append({ + "params": { + "name": items[1], + "pmem_file": items[0] + }, + "method": "bdev_pmem_create" + }) + + return pmem_json + + +def get_split_bdev_json(config, section): + split_json = [] + value = [] + for option in config.options("Split"): + if "Split" == option: + value = config.get("Split", option) + if value and not isinstance(value, list): + value = [value] + for split in value: + items = re.findall(r"\S+", split) + split_size_mb = 0 + base_bdev = items[0] + split_count = int(items[1]) + if len(items) == 3: + split_size_mb = items[2] + split_json.append({ + "params": { + "base_bdev": base_bdev, + "split_size_mb": split_size_mb, + "split_count": split_count + }, + "method": "bdev_split_create" + }) + + return split_json + + +def get_nvmf_options_json(config, section): + params = [ + ["AcceptorPollRate", "acceptor_poll_rate", int, 10000], + ["MaxSubsystems", "max_subsystems", int, 1024] + ] + for option in config.options("Nvmf"): + set_param(params, option, config.get("Nvmf", option)) + nvmf_json = [] + nvmf_json.append({ + "params": to_json_params([params[0]]), + "method": "nvmf_set_config" + }) + nvmf_json.append({ + "params": to_json_params(params[1:7]), + "method": "nvmf_set_max_subsystems" + }) + + return nvmf_json + + +def get_nvmf_subsystem_json(config, section): + nvmf_subsystem_methods = [] + params = [ + # Last items are default values if given entry is not set + ["Host", "hosts", list, []], + ["NQN", "nqn", str, ""], + ["AllowAnyHost", "allow_any_host", bool, False], + ["SN", "serial_number", str, "00000000000000000000"], + ["MN", "model_number", str, "SPDK bdev Controller"], + ["MaxNamespaces", "max_namespaces", str, ""], + ] + listen_address = [] + namespaces = [] + nsid = 0 + searched_items = [param[0] for param in params] + for option in config.options(section): + value = config.get(section, option) + if option in searched_items: + set_param(params, option, value) + continue + if "Listen" == option: + items = re.findall(r"\S+", value) + adrfam = "IPv4" + if len(items[1].split(":")) > 2: + adrfam = "IPv6" + listen_address.append({ + "trtype": items[0], + "adrfam": adrfam, + "trsvcid": items[1].rsplit(":", 1)[-1], + "traddr": items[1].rsplit(":", 1)[0].replace( + "]", "").replace("[", "") + }) + if "Namespace" == option: + for item in value.split("\n"): + items = re.findall(r"\S+", item) + if len(items) == 2: + nsid = items[1] + else: + nsid += 1 + namespaces.append({ + "nsid": int(nsid), + "bdev_name": items[0], + }) + # Get parameters: nqn, allow_any_host, serial_number + # for nvmf_create_subsystem rpc method + parameters = to_json_params(params[1:5]) + nvmf_subsystem_methods.append({ + "params": parameters, + "method": "nvmf_create_subsystem" + }) + for listen in listen_address: + nvmf_subsystem_methods.append({ + "params": { + "listen_address": listen, + "nqn": parameters['nqn'] + }, + "method": "nvmf_subsystem_add_listener" + }) + for host in to_json_params([params[0]])['hosts']: + nvmf_subsystem_methods.append({ + "params": { + "host": host, + "nqn": parameters['nqn'] + }, + "method": "nvmf_subsystem_add_host" + }) + for namespace in namespaces: + nvmf_subsystem_methods.append({ + "params": { + "namespace": namespace, + "nqn": parameters['nqn'] + }, + "method": "nvmf_subsystem_add_ns" + }) + + # Define max_namespaces if it is set in old config + if params[5][3]: + nvmf_subsystem_methods[0]['params']['max_namespaces'] = int(params[5][3]) + + return nvmf_subsystem_methods + + +def get_vhost_scsi_json(config, section): + params = [ + ["Name", "ctrlr", str, None], + ["Cpumask", "cpumask", "hex", "1"], + ] + targets = [] + vhost_scsi_json = [] + for option in config.options(section): + value = config.get(section, option) + if option in ["Name", "Cpumask"]: + set_param(params, option, value) + if "Target" == option: + for item in value.split("\n"): + items = re.findall(r"\S+", item) + targets.append({ + "scsi_target_num": int(items[0]), + "ctrlr": params[0][3], + "bdev_name": items[1] + }) + vhost_scsi_json.append({ + "params": to_json_params(params), + "method": "vhost_create_scsi_controller" + }) + for target in targets: + vhost_scsi_json.append({ + "params": target, + "method": "vhost_scsi_controller_add_target" + }) + + return vhost_scsi_json + + +def get_vhost_blk_json(config, section): + params = [ + ["ReadOnly", "readonly", bool, False], + ["Dev", "dev_name", str, ""], + ["Name", "ctrlr", str, ""], + ["Cpumask", "cpumask", "hex", ""] + ] + for option in config.options(section): + set_param(params, option, config.get(section, option)) + return [{"method": "vhost_create_blk_controller", + "params": to_json_params(params)}] + + +def get_vhost_nvme_json(config, section): + params = [ + ["Name", "ctrlr", str, ""], + ["NumberOfQueues", "io_queues", int, -1], + ["Cpumask", "cpumask", "hex", 0x1], + ["Namespace", "bdev_name", list, []] + ] + for option in config.options(section): + values = config.get(section, option).split("\n") + for value in values: + set_param(params, option, value) + vhost_nvme_json = [] + vhost_nvme_json.append({ + "params": to_json_params(params[:3]), + "method": "vhost_create_nvme_controller" + }) + for namespace in params[3][3]: + vhost_nvme_json.append({ + "params": { + "ctrlr": params[0][3], + "bdev_name": namespace, + }, + "method": "vhost_nvme_controller_add_ns" + }) + + return vhost_nvme_json + + +def get_virtio_user_json(config, section): + params = [ + ["Path", "traddr", str, ""], + ["Queues", "vq_count", int, 1], + ["Type", "dev_type", "dev_type", "scsi"], + ["Name", "name", str, section], + # Define parameters with default values. + # These params are set by rpc commands and + # do not occur in ini config file. + # But they are visible in json config file + # with default values even if not set by rpc. + [None, "trtype", str, "user"], + [None, "vq_size", int, 512] + ] + for option in config.options(section): + set_param(params, option, config.get(section, option)) + dev_name = "Scsi" + if params[2][3] == "blk": + dev_name = "Blk" + params[3][3] = params[3][3].replace("User", dev_name) + + return [{ + "params": to_json_params(params), + "method": "bdev_virtio_attach_controller" + }] + + +def get_iscsi_options_json(config, section): + params = [ + ['AllowDuplicateIsid', 'allow_duplicated_isid', bool, False], + ['DefaultTime2Retain', 'default_time2retain', int, 20], + ['DiscoveryAuthMethod', 'mutual_chap', bool, False], + ['MaxConnectionsPerSession', 'max_connections_per_session', int, 2], + ['Timeout', 'nop_timeout', int, 60], + ['DiscoveryAuthMethod', 'disable_chap', bool, False], + ['DiscoveryAuthMethod', 'require_chap', bool, False], + ['NodeBase', 'node_base', str, "iqn.2016-06.io.spdk"], + ['AuthFile', 'auth_file', str, None], + ['DiscoveryAuthGroup', 'chap_group', int, 0], + ['MaxSessions', 'max_sessions', int, 128], + ['ImmediateData', 'immediate_data', bool, True], + ['ErrorRecoveryLevel', 'error_recovery_level', int, 0], + ['NopInInterval', 'nop_in_interval', int, 30], + ['DefaultTime2Wait', 'default_time2wait', int, 2], + ['QueueDepth', 'max_queue_depth', int, 64], + ['', 'first_burst_length', int, 8192] + ] + for option in config.options(section): + set_param(params, option, config.get(section, option)) + return [{"method": "iscsi_set_options", "params": to_json_params(params)}] + + +def get_iscsi_portal_group_json(config, name): + portal_group_json = [] + portals = [] + for option in config.options(name): + if "Portal" == option: + for value in config.get(name, option).split("\n"): + items = re.findall(r"\S+", value) + portal = {'host': items[1].rsplit(":", 1)[0]} + if "@" in items[1]: + portal['port'] =\ + items[1].rsplit(":", 1)[1].split("@")[0] + else: + portal['port'] = items[1].rsplit(":", 1)[1] + portals.append(portal) + + portal_group_json.append({ + "params": { + "portals": portals, + "tag": int(re.findall(r'\d+', name)[0]) + }, + "method": "iscsi_create_portal_group" + }) + + return portal_group_json + + +def get_iscsi_initiator_group_json(config, name): + initiators = [] + netmasks = [] + + for option in config.options(name): + if "InitiatorName" == option: + initiators.append(config.get(name, option)) + if "Netmask" == option: + netmasks.append(config.get(name, option)) + initiator_group_json = { + "params": { + "initiators": initiators, + "tag": int(re.findall(r'\d+', name)[0]), + "netmasks": netmasks + }, + "method": "iscsi_create_initiator_group" + } + + return [initiator_group_json] + + +def get_iscsi_target_node_json(config, section): + luns = [] + mutual_chap = False + name = "" + alias_name = "" + require_chap = False + chap_group = 1 + pg_ig_maps = [] + data_digest = False + disable_chap = False + header_digest = False + queue_depth = 64 + + for option in config.options(section): + value = config.get(section, option) + if "TargetName" == option: + name = value + if "TargetAlias" == option: + alias_name = value.replace("\"", "") + if "Mapping" == option: + items = re.findall(r"\S+", value) + pg_ig_maps.append({ + "ig_tag": int(re.findall(r'\d+', items[1])[0]), + "pg_tag": int(re.findall(r'\d+', items[0])[0]) + }) + if "AuthMethod" == option: + items = re.findall(r"\S+", value) + for item in items: + if "CHAP" == item: + require_chap = True + elif "Mutual" == item: + mutual_chap = True + elif "Auto" == item: + disable_chap = False + require_chap = False + mutual_chap = False + elif "None" == item: + disable_chap = True + require_chap = False + mutual_chap = False + if "AuthGroup" == option: # AuthGroup1 + items = re.findall(r"\S+", value) + chap_group = int(re.findall(r'\d+', items[0])[0]) + if "UseDigest" == option: + items = re.findall(r"\S+", value) + for item in items: + if "Header" == item: + header_digest = True + elif "Data" == item: + data_digest = True + elif "Auto" == item: + header_digest = False + data_digest = False + + if re.match(r"LUN\d+", option): + luns.append({"lun_id": len(luns), + "bdev_name": value}) + if "QueueDepth" == option: + queue_depth = int(value) + + params = {"alias_name": alias_name} + params["name"] = "iqn.2016-06.io.spdk:%s" % name + params["luns"] = luns + params["pg_ig_maps"] = pg_ig_maps + params["queue_depth"] = queue_depth + params["chap_group"] = chap_group + params["header_digest"] = header_digest + params["mutual_chap"] = mutual_chap + params["require_chap"] = require_chap + params["data_digest"] = data_digest + params["disable_chap"] = disable_chap + + target_json = { + "params": params, + "method": "iscsi_create_target_node" + } + + return [target_json] + + +if __name__ == "__main__": + try: + config = configparser.ConfigParser(strict=False, delimiters=(' '), + dict_type=OptionOrderedDict, + allow_no_value=True) + # Do not parse options and values. Capital letters are relevant. + config.optionxform = str + config.read_file(sys.stdin) + except Exception as e: + print("Exception while parsing config: %s" % e) + exit(1) + # Add missing sections to generate default configuration + for section in ['Nvme', 'Nvmf', 'Bdev', 'iSCSI']: + if section not in config.sections(): + config.add_section(section) + + for section in config.sections(): + match = re.match(r'(Bdev|Nvme|Malloc|VirtioUser\d+|Split|Pmem|AIO|' + r'iSCSI|PortalGroup\d+|InitiatorGroup\d+|' + r'TargetNode\d+|Nvmf|Subsystem\d+|VhostScsi\d+|' + r'VhostBlk\d+|VhostNvme\d+)', section) + if match: + match_section = ''.join(letter for letter in match.group(0) + if not letter.isdigit()) + if match_section == "Bdev": + items = get_bdev_options_json(config, section) + elif match_section == "AIO": + items = get_aio_bdev_json(config, section) + elif match_section == "Malloc": + items = get_malloc_bdev_json(config, section) + elif match_section == "Nvme": + items = get_nvme_bdev_json(config, section) + elif match_section == "Pmem": + items = get_pmem_bdev_json(config, section) + elif match_section == "Split": + items = get_split_bdev_json(config, section) + elif match_section == "Nvmf": + items = get_nvmf_options_json(config, section) + elif match_section == "Subsystem": + items = get_nvmf_subsystem_json(config, section) + elif match_section == "VhostScsi": + items = get_vhost_scsi_json(config, section) + elif match_section == "VhostBlk": + items = get_vhost_blk_json(config, section) + elif match_section == "VhostNvme": + items = get_vhost_nvme_json(config, section) + elif match_section == "VirtioUser": + items = get_virtio_user_json(config, section) + elif match_section == "iSCSI": + items = get_iscsi_options_json(config, section) + elif match_section == "PortalGroup": + items = get_iscsi_portal_group_json(config, section) + elif match_section == "InitiatorGroup": + items = get_iscsi_initiator_group_json(config, section) + elif match_section == "TargetNode": + items = get_iscsi_target_node_json(config, section) + for item in items: + if match_section == "VhostScsi": + section_to_subsystem[match_section]["vhost_create_scsi_controller"].append(item) + elif match_section == "VhostNvme": + section_to_subsystem[match_section]["vhost_create_nvme_controller"].append(item) + elif match_section == "Subsystem": + section_to_subsystem[match_section]["subsystems"].append(item) + else: + section_to_subsystem[match_section][ + item['method']].append(item) + elif section == "Global": + pass + elif section == "Ioat": + # Ioat doesn't support JSON config yet. + pass + elif section == "VirtioPci": + print("Please use spdk target flags.") + exit(1) + else: + print("An invalid section detected: %s.\n" + "Please revise your config file." % section) + exit(1) + json.dump(generate_new_json_config(), sys.stdout, indent=2) + print("") |