diff options
Diffstat (limited to 'src/spdk/lib/vmd')
| -rw-r--r-- | src/spdk/lib/vmd/Makefile | 45 | ||||
| -rw-r--r-- | src/spdk/lib/vmd/led.c | 166 | ||||
| -rw-r--r-- | src/spdk/lib/vmd/spdk_vmd.map | 13 | ||||
| -rw-r--r-- | src/spdk/lib/vmd/vmd.c | 1376 | ||||
| -rw-r--r-- | src/spdk/lib/vmd/vmd.h | 201 | ||||
| -rw-r--r-- | src/spdk/lib/vmd/vmd_spec.h | 473 |
6 files changed, 2274 insertions, 0 deletions
diff --git a/src/spdk/lib/vmd/Makefile b/src/spdk/lib/vmd/Makefile new file mode 100644 index 000000000..13813c559 --- /dev/null +++ b/src/spdk/lib/vmd/Makefile @@ -0,0 +1,45 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +SO_VER := 2 +SO_MINOR := 0 + +C_SRCS = vmd.c led.c +LIBNAME = vmd + +SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_vmd.map) + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/src/spdk/lib/vmd/led.c b/src/spdk/lib/vmd/led.c new file mode 100644 index 000000000..878983aab --- /dev/null +++ b/src/spdk/lib/vmd/led.c @@ -0,0 +1,166 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/stdinc.h" +#include "spdk/likely.h" +#include "spdk/log.h" +#include "vmd.h" + +struct vmd_led_indicator_config { + uint8_t attention_indicator : 2; + uint8_t power_indicator : 2; + uint8_t reserved : 4; +}; + +/* + * VMD LED Attn Power LED Amber + * State Indicator Indicator + * Control Control + * ------------------------------------------------ + * Off 11b 11b Off + * Ident 11b 01b Blink 4Hz + * Fault 01b 11b On + * Rebuild 01b 01b Blink 1Hz + */ +static const struct vmd_led_indicator_config g_led_config[] = { + [SPDK_VMD_LED_STATE_OFF] = { .attention_indicator = 3, .power_indicator = 3 }, + [SPDK_VMD_LED_STATE_IDENTIFY] = { .attention_indicator = 3, .power_indicator = 1 }, + [SPDK_VMD_LED_STATE_FAULT] = { .attention_indicator = 1, .power_indicator = 3 }, + [SPDK_VMD_LED_STATE_REBUILD] = { .attention_indicator = 1, .power_indicator = 1 }, +}; + +static void +vmd_led_set_indicator_control(struct vmd_pci_device *vmd_device, enum spdk_vmd_led_state state) +{ + const struct vmd_led_indicator_config *config; + union express_slot_control_register slot_control; + + assert(state >= SPDK_VMD_LED_STATE_OFF && state <= SPDK_VMD_LED_STATE_REBUILD); + config = &g_led_config[state]; + + slot_control = vmd_device->pcie_cap->slot_control; + slot_control.bit_field.attention_indicator_control = config->attention_indicator; + slot_control.bit_field.power_indicator_control = config->power_indicator; + + /* + * Due to the fact that writes to the PCI config space are posted writes, we need to issue + * a read to the register we've just written to ensure it reached its destination. + * TODO: wrap all register writes with a function taking care of that. + */ + vmd_device->pcie_cap->slot_control = slot_control; + vmd_device->cached_slot_control = vmd_device->pcie_cap->slot_control; +} + +static unsigned int +vmd_led_get_state(struct vmd_pci_device *vmd_device) +{ + const struct vmd_led_indicator_config *config; + union express_slot_control_register slot_control; + unsigned int state; + + slot_control = vmd_device->cached_slot_control; + for (state = SPDK_VMD_LED_STATE_OFF; state <= SPDK_VMD_LED_STATE_REBUILD; ++state) { + config = &g_led_config[state]; + + if (slot_control.bit_field.attention_indicator_control == config->attention_indicator && + slot_control.bit_field.power_indicator_control == config->power_indicator) { + return state; + } + } + + return SPDK_VMD_LED_STATE_UNKNOWN; +} + +/* + * The identifying device under VMD is located in the global list of VMD controllers. If the BDF + * identifies an endpoint, then the LED is attached to the endpoint's parent. If the BDF identifies + * a type 1 header, then this device has the corresponding LED. This may arise when a user wants to + * identify a given empty slot under VMD. + */ +static struct vmd_pci_device * +vmd_get_led_device(const struct spdk_pci_device *pci_device) +{ + struct vmd_pci_device *vmd_device; + + assert(strcmp(spdk_pci_device_get_type(pci_device), "vmd") == 0); + + vmd_device = vmd_find_device(&pci_device->addr); + if (spdk_unlikely(vmd_device == NULL)) { + return NULL; + } + + if (vmd_device->header_type == PCI_HEADER_TYPE_NORMAL) { + if (spdk_unlikely(vmd_device->parent == NULL)) { + return NULL; + } + + return vmd_device->parent->self; + } + + return vmd_device; +} + +int +spdk_vmd_set_led_state(struct spdk_pci_device *pci_device, enum spdk_vmd_led_state state) +{ + struct vmd_pci_device *vmd_device; + + if (state < SPDK_VMD_LED_STATE_OFF || state > SPDK_VMD_LED_STATE_REBUILD) { + SPDK_ERRLOG("Invalid LED state\n"); + return -EINVAL; + } + + vmd_device = vmd_get_led_device(pci_device); + if (spdk_unlikely(vmd_device == NULL)) { + SPDK_ERRLOG("The PCI device is not behind the VMD\n"); + return -ENODEV; + } + + vmd_led_set_indicator_control(vmd_device, state); + return 0; +} + +int +spdk_vmd_get_led_state(struct spdk_pci_device *pci_device, enum spdk_vmd_led_state *state) +{ + struct vmd_pci_device *vmd_device; + + vmd_device = vmd_get_led_device(pci_device); + if (spdk_unlikely(vmd_device == NULL)) { + SPDK_ERRLOG("The PCI device is not behind the VMD\n"); + return -ENODEV; + } + + *state = (enum spdk_vmd_led_state)vmd_led_get_state(vmd_device); + return 0; +} diff --git a/src/spdk/lib/vmd/spdk_vmd.map b/src/spdk/lib/vmd/spdk_vmd.map new file mode 100644 index 000000000..036d079b5 --- /dev/null +++ b/src/spdk/lib/vmd/spdk_vmd.map @@ -0,0 +1,13 @@ +{ + global: + + # public functions + spdk_vmd_init; + spdk_vmd_fini; + spdk_vmd_pci_device_list; + spdk_vmd_set_led_state; + spdk_vmd_get_led_state; + spdk_vmd_hotplug_monitor; + + local: *; +}; diff --git a/src/spdk/lib/vmd/vmd.c b/src/spdk/lib/vmd/vmd.c new file mode 100644 index 000000000..14d9558c2 --- /dev/null +++ b/src/spdk/lib/vmd/vmd.c @@ -0,0 +1,1376 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "vmd.h" + +#include "spdk/stdinc.h" +#include "spdk/likely.h" + +static unsigned char *device_type[] = { + "PCI Express Endpoint", + "Legacy PCI Express Endpoint", + "Reserved 1", + "Reserved 2", + "Root Port of PCI Express Root Complex", + "Upstream Port of PCI Express Switch", + "Downstream Port of PCI Express Switch", + "PCI Express to PCI/PCI-X Bridge", + "PCI/PCI-X to PCI Express Bridge", + "Root Complex Integrated Endpoint", + "Root Complex Event Collector", + "Reserved Capability" +}; + +/* + * Container for all VMD adapter probed in the system. + */ +struct vmd_container { + uint32_t count; + struct vmd_adapter vmd[MAX_VMD_SUPPORTED]; +}; + +static struct vmd_container g_vmd_container; +static uint8_t g_end_device_count; + +static bool +vmd_is_valid_cfg_addr(struct vmd_pci_bus *bus, uint64_t addr) +{ + return addr >= (uint64_t)bus->vmd->cfg_vaddr && + addr < bus->vmd->cfgbar_size + (uint64_t)bus->vmd->cfg_vaddr; +} + +static void +vmd_align_base_addrs(struct vmd_adapter *vmd, uint32_t alignment) +{ + uint32_t pad; + + /* + * Device is not in hot plug path, align the base address remaining from membar 1. + */ + if (vmd->physical_addr & (alignment - 1)) { + pad = alignment - (vmd->physical_addr & (alignment - 1)); + vmd->physical_addr += pad; + vmd->current_addr_size -= pad; + } +} + +static bool +vmd_device_is_enumerated(const struct vmd_pci_device *vmd_device) +{ + return vmd_device->header->one.prefetch_base_upper == VMD_UPPER_BASE_SIGNATURE && + vmd_device->header->one.prefetch_limit_upper == VMD_UPPER_LIMIT_SIGNATURE; +} + +static bool +vmd_device_is_root_port(const struct vmd_pci_device *vmd_device) +{ + return vmd_device->header->common.vendor_id == 0x8086 && + (vmd_device->header->common.device_id == 0x2030 || + vmd_device->header->common.device_id == 0x2031 || + vmd_device->header->common.device_id == 0x2032 || + vmd_device->header->common.device_id == 0x2033); +} + +static void +vmd_hotplug_coalesce_regions(struct vmd_hot_plug *hp) +{ + struct pci_mem_mgr *region, *prev; + + do { + prev = NULL; + TAILQ_FOREACH(region, &hp->free_mem_queue, tailq) { + if (prev != NULL && (prev->addr + prev->size == region->addr)) { + break; + } + + prev = region; + } + + if (region != NULL) { + prev->size += region->size; + TAILQ_REMOVE(&hp->free_mem_queue, region, tailq); + TAILQ_INSERT_TAIL(&hp->unused_mem_queue, region, tailq); + } + } while (region != NULL); +} + +static void +vmd_hotplug_free_region(struct vmd_hot_plug *hp, struct pci_mem_mgr *region) +{ + struct pci_mem_mgr *current, *prev = NULL; + + assert(region->addr >= hp->bar.start && region->addr < hp->bar.start + hp->bar.size); + + TAILQ_FOREACH(current, &hp->free_mem_queue, tailq) { + if (current->addr > region->addr) { + break; + } + + prev = current; + } + + if (prev != NULL) { + assert(prev->addr + prev->size <= region->addr); + assert(current == NULL || (region->addr + region->size <= current->addr)); + TAILQ_INSERT_AFTER(&hp->free_mem_queue, prev, region, tailq); + } else { + TAILQ_INSERT_HEAD(&hp->free_mem_queue, region, tailq); + } + + vmd_hotplug_coalesce_regions(hp); +} + +static void +vmd_hotplug_free_addr(struct vmd_hot_plug *hp, uint64_t addr) +{ + struct pci_mem_mgr *region; + + TAILQ_FOREACH(region, &hp->alloc_mem_queue, tailq) { + if (region->addr == addr) { + break; + } + } + + assert(region != NULL); + TAILQ_REMOVE(&hp->alloc_mem_queue, region, tailq); + + vmd_hotplug_free_region(hp, region); +} + +static uint64_t +vmd_hotplug_allocate_base_addr(struct vmd_hot_plug *hp, uint32_t size) +{ + struct pci_mem_mgr *region = NULL, *free_region; + + TAILQ_FOREACH(region, &hp->free_mem_queue, tailq) { + if (region->size >= size) { + break; + } + } + + if (region == NULL) { + SPDK_DEBUGLOG(SPDK_LOG_VMD, "Unable to find free hotplug memory region of size:" + "%"PRIx32"\n", size); + return 0; + } + + TAILQ_REMOVE(&hp->free_mem_queue, region, tailq); + if (size < region->size) { + free_region = TAILQ_FIRST(&hp->unused_mem_queue); + if (free_region == NULL) { + SPDK_DEBUGLOG(SPDK_LOG_VMD, "Unable to find unused descriptor to store the " + "free region of size: %"PRIu32"\n", region->size - size); + } else { + TAILQ_REMOVE(&hp->unused_mem_queue, free_region, tailq); + free_region->size = region->size - size; + free_region->addr = region->addr + size; + region->size = size; + vmd_hotplug_free_region(hp, free_region); + } + } + + TAILQ_INSERT_TAIL(&hp->alloc_mem_queue, region, tailq); + + return region->addr; +} + +/* + * Allocates an address from vmd membar for the input memory size + * vmdAdapter - vmd adapter object + * dev - vmd_pci_device to allocate a base address for. + * size - size of the memory window requested. + * Size must be an integral multiple of 2. Addresses are returned on the size boundary. + * Returns physical address within the VMD membar window, or 0x0 if cannot allocate window. + * Consider increasing the size of vmd membar if 0x0 is returned. + */ +static uint64_t +vmd_allocate_base_addr(struct vmd_adapter *vmd, struct vmd_pci_device *dev, uint32_t size) +{ + uint64_t base_address = 0, padding = 0; + struct vmd_pci_bus *hp_bus; + + if (size && ((size & (~size + 1)) != size)) { + return base_address; + } + + /* + * If device is downstream of a hot plug port, allocate address from the + * range dedicated for the hot plug slot. Search the list of addresses allocated to determine + * if a free range exists that satisfy the input request. If a free range cannot be found, + * get a buffer from the unused chunk. First fit algorithm, is used. + */ + if (dev) { + hp_bus = dev->parent; + if (hp_bus && hp_bus->self && hp_bus->self->hotplug_capable) { + return vmd_hotplug_allocate_base_addr(&hp_bus->self->hp, size); + } + } + + /* Ensure physical membar allocated is size aligned */ + if (vmd->physical_addr & (size - 1)) { + padding = size - (vmd->physical_addr & (size - 1)); + } + + /* Allocate from membar if enough memory is left */ + if (vmd->current_addr_size >= size + padding) { + base_address = vmd->physical_addr + padding; + vmd->physical_addr += size + padding; + vmd->current_addr_size -= size + padding; + } + + SPDK_DEBUGLOG(SPDK_LOG_VMD, "allocated(size) %lx (%x)\n", base_address, size); + + return base_address; +} + +static bool +vmd_is_end_device(struct vmd_pci_device *dev) +{ + return (dev && dev->header) && + ((dev->header->common.header_type & ~PCI_MULTI_FUNCTION) == PCI_HEADER_TYPE_NORMAL); +} + +static void +vmd_update_base_limit_register(struct vmd_pci_device *dev, uint16_t base, uint16_t limit) +{ + struct vmd_pci_bus *bus; + struct vmd_pci_device *bridge; + + if (base == 0 || limit == 0) { + return; + } + + if (dev->header->common.header_type == PCI_HEADER_TYPE_BRIDGE) { + bus = dev->bus_object; + } else { + bus = dev->parent; + } + + bridge = bus->self; + SPDK_DEBUGLOG(SPDK_LOG_VMD, "base:limit = %x:%x\n", bridge->header->one.mem_base, + bridge->header->one.mem_limit); + + if (dev->bus->vmd->scan_completed) { + return; + } + + while (bus && bus->self != NULL) { + bridge = bus->self; + + /* This is only for 32-bit memory space, need to revisit to support 64-bit */ + if (bridge->header->one.mem_base > base) { + bridge->header->one.mem_base = base; + base = bridge->header->one.mem_base; + } + + if (bridge->header->one.mem_limit < limit) { + bridge->header->one.mem_limit = limit; + limit = bridge->header->one.mem_limit; + } + + bus = bus->parent; + } +} + +static uint64_t +vmd_get_base_addr(struct vmd_pci_device *dev, uint32_t index, uint32_t size) +{ + struct vmd_pci_bus *bus = dev->parent; + + if (dev->header_type == PCI_HEADER_TYPE_BRIDGE) { + return dev->header->zero.BAR[index] & ~0xf; + } else { + if (bus->self->hotplug_capable) { + return vmd_hotplug_allocate_base_addr(&bus->self->hp, size); + } else { + return (uint64_t)bus->self->header->one.mem_base << 16; + } + } +} + +static bool +vmd_assign_base_addrs(struct vmd_pci_device *dev) +{ + uint16_t mem_base = 0, mem_limit = 0; + unsigned char mem_attr = 0; + int last; + struct vmd_adapter *vmd = NULL; + bool ret_val = false; + uint32_t bar_value; + uint32_t table_offset; + + if (dev && dev->bus) { + vmd = dev->bus->vmd; + } + + if (!vmd) { + return 0; + } + + vmd_align_base_addrs(vmd, ONE_MB); + + last = dev->header_type ? 2 : 6; + for (int i = 0; i < last; i++) { + bar_value = dev->header->zero.BAR[i]; + dev->header->zero.BAR[i] = ~(0U); + dev->bar[i].size = dev->header->zero.BAR[i]; + dev->header->zero.BAR[i] = bar_value; + + if (dev->bar[i].size == ~(0U) || dev->bar[i].size == 0 || + dev->header->zero.BAR[i] & 1) { + dev->bar[i].size = 0; + continue; + } + mem_attr = dev->bar[i].size & PCI_BASE_ADDR_MASK; + dev->bar[i].size = TWOS_COMPLEMENT(dev->bar[i].size & PCI_BASE_ADDR_MASK); + + if (vmd->scan_completed) { + dev->bar[i].start = vmd_get_base_addr(dev, i, dev->bar[i].size); + } else { + dev->bar[i].start = vmd_allocate_base_addr(vmd, dev, dev->bar[i].size); + } + + dev->header->zero.BAR[i] = (uint32_t)dev->bar[i].start; + + if (!dev->bar[i].start) { + if (mem_attr == (PCI_BAR_MEMORY_PREFETCH | PCI_BAR_MEMORY_TYPE_64)) { + i++; + } + continue; + } + + dev->bar[i].vaddr = ((uint64_t)vmd->mem_vaddr + (dev->bar[i].start - vmd->membar)); + mem_limit = BRIDGE_BASEREG(dev->header->zero.BAR[i]) + + BRIDGE_BASEREG(dev->bar[i].size - 1); + if (!mem_base) { + mem_base = BRIDGE_BASEREG(dev->header->zero.BAR[i]); + } + + ret_val = true; + + if (mem_attr == (PCI_BAR_MEMORY_PREFETCH | PCI_BAR_MEMORY_TYPE_64)) { + i++; + if (i < last) { + dev->header->zero.BAR[i] = (uint32_t)(dev->bar[i].start >> PCI_DWORD_SHIFT); + } + } + } + + /* Enable device MEM and bus mastering */ + dev->header->zero.command |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + uint16_t cmd = dev->header->zero.command; + cmd++; + + if (dev->msix_cap && ret_val) { + table_offset = ((volatile struct pci_msix_cap *)dev->msix_cap)->msix_table_offset; + if (dev->bar[table_offset & 0x3].vaddr) { + dev->msix_table = (volatile struct pci_msix_table_entry *) + (dev->bar[table_offset & 0x3].vaddr + (table_offset & 0xfff8)); + } + } + + if (ret_val && vmd_is_end_device(dev)) { + vmd_update_base_limit_register(dev, mem_base, mem_limit); + } + + return ret_val; +} + +static void +vmd_get_device_capabilities(struct vmd_pci_device *dev) + +{ + volatile uint8_t *config_space; + uint8_t capabilities_offset; + struct pci_capabilities_header *capabilities_hdr; + + config_space = (volatile uint8_t *)dev->header; + if ((dev->header->common.status & PCI_CAPABILITIES_LIST) == 0) { + return; + } + + capabilities_offset = dev->header->zero.cap_pointer; + if (dev->header->common.header_type & PCI_HEADER_TYPE_BRIDGE) { + capabilities_offset = dev->header->one.cap_pointer; + } + + while (capabilities_offset > 0) { + capabilities_hdr = (struct pci_capabilities_header *) + &config_space[capabilities_offset]; + switch (capabilities_hdr->capability_id) { + case CAPABILITY_ID_PCI_EXPRESS: + dev->pcie_cap = (volatile struct pci_express_cap *)(capabilities_hdr); + break; + + case CAPABILITY_ID_MSI: + dev->msi_cap = (volatile struct pci_msi_cap *)capabilities_hdr; + break; + + case CAPABILITY_ID_MSIX: + dev->msix_cap = (volatile struct pci_msix_capability *)capabilities_hdr; + dev->msix_table_size = dev->msix_cap->message_control.bit.table_size + 1; + break; + + default: + break; + } + capabilities_offset = capabilities_hdr->next; + } +} + +static volatile struct pci_enhanced_capability_header * +vmd_get_enhanced_capabilities(struct vmd_pci_device *dev, uint16_t capability_id) +{ + uint8_t *data; + uint16_t cap_offset = EXTENDED_CAPABILITY_OFFSET; + volatile struct pci_enhanced_capability_header *cap_hdr = NULL; + + data = (uint8_t *)dev->header; + while (cap_offset >= EXTENDED_CAPABILITY_OFFSET) { + cap_hdr = (volatile struct pci_enhanced_capability_header *) &data[cap_offset]; + if (cap_hdr->capability_id == capability_id) { + return cap_hdr; + } + cap_offset = cap_hdr->next; + if (cap_offset == 0 || cap_offset < EXTENDED_CAPABILITY_OFFSET) { + break; + } + } + + return NULL; +} + +static void +vmd_read_config_space(struct vmd_pci_device *dev) +{ + /* + * Writes to the pci config space is posted weite. To ensure transaction reaches its destination + * before another write is posed, an immediate read of the written value should be performed. + */ + dev->header->common.command |= (BUS_MASTER_ENABLE | MEMORY_SPACE_ENABLE); + { uint16_t cmd = dev->header->common.command; (void)cmd; } + + vmd_get_device_capabilities(dev); + dev->sn_cap = (struct serial_number_capability *)vmd_get_enhanced_capabilities(dev, + DEVICE_SERIAL_NUMBER_CAP_ID); +} + +static void +vmd_update_scan_info(struct vmd_pci_device *dev) +{ + struct vmd_adapter *vmd_adapter = dev->bus->vmd; + + if (vmd_adapter->root_port_updated) { + return; + } + + if (dev->header_type == PCI_HEADER_TYPE_NORMAL) { + return; + } + + if (vmd_device_is_root_port(dev)) { + vmd_adapter->root_port_updated = 1; + SPDK_DEBUGLOG(SPDK_LOG_VMD, "root_port_updated = %d\n", + vmd_adapter->root_port_updated); + SPDK_DEBUGLOG(SPDK_LOG_VMD, "upper:limit = %x : %x\n", + dev->header->one.prefetch_base_upper, + dev->header->one.prefetch_limit_upper); + if (vmd_device_is_enumerated(dev)) { + vmd_adapter->scan_completed = 1; + SPDK_DEBUGLOG(SPDK_LOG_VMD, "scan_completed = %d\n", + vmd_adapter->scan_completed); + } + } +} + +static void +vmd_reset_base_limit_registers(struct vmd_pci_device *dev) +{ + uint32_t reg __attribute__((unused)); + + assert(dev->header_type != PCI_HEADER_TYPE_NORMAL); + /* + * Writes to the pci config space are posted writes. + * To ensure transaction reaches its destination + * before another write is posted, an immediate read + * of the written value should be performed. + */ + dev->header->one.mem_base = 0xfff0; + reg = dev->header->one.mem_base; + dev->header->one.mem_limit = 0x0; + reg = dev->header->one.mem_limit; + dev->header->one.prefetch_base = 0x0; + reg = dev->header->one.prefetch_base; + dev->header->one.prefetch_limit = 0x0; + reg = dev->header->one.prefetch_limit; + dev->header->one.prefetch_base_upper = 0x0; + reg = dev->header->one.prefetch_base_upper; + dev->header->one.prefetch_limit_upper = 0x0; + reg = dev->header->one.prefetch_limit_upper; + dev->header->one.io_base_upper = 0x0; + reg = dev->header->one.io_base_upper; + dev->header->one.io_limit_upper = 0x0; + reg = dev->header->one.io_limit_upper; + dev->header->one.primary = 0; + reg = dev->header->one.primary; + dev->header->one.secondary = 0; + reg = dev->header->one.secondary; + dev->header->one.subordinate = 0; + reg = dev->header->one.subordinate; +} + +static void +vmd_init_hotplug(struct vmd_pci_device *dev, struct vmd_pci_bus *bus) +{ + struct vmd_adapter *vmd = bus->vmd; + struct vmd_hot_plug *hp = &dev->hp; + size_t mem_id; + + dev->hotplug_capable = true; + hp->bar.size = 1 << 20; + + if (!vmd->scan_completed) { + hp->bar.start = vmd_allocate_base_addr(vmd, NULL, hp->bar.size); + bus->self->header->one.mem_base = BRIDGE_BASEREG(hp->bar.start); + bus->self->header->one.mem_limit = + bus->self->header->one.mem_base + BRIDGE_BASEREG(hp->bar.size - 1); + } else { + hp->bar.start = (uint64_t)bus->self->header->one.mem_base << 16; + } + + hp->bar.vaddr = (uint64_t)vmd->mem_vaddr + (hp->bar.start - vmd->membar); + + TAILQ_INIT(&hp->free_mem_queue); + TAILQ_INIT(&hp->unused_mem_queue); + TAILQ_INIT(&hp->alloc_mem_queue); + + hp->mem[0].size = hp->bar.size; + hp->mem[0].addr = hp->bar.start; + + TAILQ_INSERT_TAIL(&hp->free_mem_queue, &hp->mem[0], tailq); + + for (mem_id = 1; mem_id < ADDR_ELEM_COUNT; ++mem_id) { + TAILQ_INSERT_TAIL(&hp->unused_mem_queue, &hp->mem[mem_id], tailq); + } + + SPDK_DEBUGLOG(SPDK_LOG_VMD, "%s: mem_base:mem_limit = %x : %x\n", __func__, + bus->self->header->one.mem_base, bus->self->header->one.mem_limit); +} + +static bool +vmd_bus_device_present(struct vmd_pci_bus *bus, uint32_t devfn) +{ + volatile struct pci_header *header; + + header = (volatile struct pci_header *)(bus->vmd->cfg_vaddr + + CONFIG_OFFSET_ADDR(bus->bus_number, devfn, 0, 0)); + if (!vmd_is_valid_cfg_addr(bus, (uint64_t)header)) { + return false; + } + + if (header->common.vendor_id == PCI_INVALID_VENDORID || header->common.vendor_id == 0x0) { + return false; + } + + return true; +} + +static struct vmd_pci_device * +vmd_alloc_dev(struct vmd_pci_bus *bus, uint32_t devfn) +{ + struct vmd_pci_device *dev = NULL; + struct pci_header volatile *header; + uint8_t header_type; + uint32_t rev_class; + + /* Make sure we're not creating two devices on the same dev/fn */ + TAILQ_FOREACH(dev, &bus->dev_list, tailq) { + if (dev->devfn == devfn) { + return NULL; + } + } + + if (!vmd_bus_device_present(bus, devfn)) { + return NULL; + } + + header = (struct pci_header * volatile)(bus->vmd->cfg_vaddr + + CONFIG_OFFSET_ADDR(bus->bus_number, devfn, 0, 0)); + + SPDK_DEBUGLOG(SPDK_LOG_VMD, "PCI device found: %04x:%04x ***\n", + header->common.vendor_id, header->common.device_id); + + dev = calloc(1, sizeof(*dev)); + if (!dev) { + return NULL; + } + + dev->header = header; + dev->vid = dev->header->common.vendor_id; + dev->did = dev->header->common.device_id; + dev->bus = bus; + dev->parent = bus; + dev->devfn = devfn; + header_type = dev->header->common.header_type; + rev_class = dev->header->common.rev_class; + dev->class = rev_class >> 8; + dev->header_type = header_type & 0x7; + + if (header_type == PCI_HEADER_TYPE_BRIDGE) { + vmd_update_scan_info(dev); + if (!dev->bus->vmd->scan_completed) { + vmd_reset_base_limit_registers(dev); + } + } + + vmd_read_config_space(dev); + + return dev; +} + +static struct vmd_pci_bus * +vmd_create_new_bus(struct vmd_pci_bus *parent, struct vmd_pci_device *bridge, uint8_t bus_number) +{ + struct vmd_pci_bus *new_bus; + + new_bus = calloc(1, sizeof(*new_bus)); + if (!new_bus) { + return NULL; + } + + new_bus->parent = parent; + new_bus->domain = parent->domain; + new_bus->bus_number = bus_number; + new_bus->secondary_bus = new_bus->subordinate_bus = bus_number; + new_bus->self = bridge; + new_bus->vmd = parent->vmd; + TAILQ_INIT(&new_bus->dev_list); + + bridge->subordinate = new_bus; + + bridge->pci.addr.bus = new_bus->bus_number; + bridge->pci.addr.dev = bridge->devfn; + bridge->pci.addr.func = 0; + bridge->pci.addr.domain = parent->vmd->pci->addr.domain; + + return new_bus; +} + +/* + * Assigns a bus number from the list of available + * bus numbers. If the device is downstream of a hot plug port, + * assign the bus number from thiose assigned to the HP port. Otherwise, + * assign the next bus number from the vmd bus number list. + */ +static uint8_t +vmd_get_next_bus_number(struct vmd_pci_device *dev, struct vmd_adapter *vmd) +{ + uint8_t bus = 0xff; + struct vmd_pci_bus *hp_bus; + + if (dev) { + hp_bus = vmd_is_dev_in_hotplug_path(dev); + if (hp_bus && hp_bus->self && hp_bus->self->hotplug_capable) { + return vmd_hp_get_next_bus_number(&hp_bus->self->hp); + } + } + + /* Device is not under a hot plug path. Return next global bus number */ + if ((vmd->next_bus_number + 1) < vmd->max_pci_bus) { + bus = vmd->next_bus_number; + vmd->next_bus_number++; + } + return bus; +} + +static uint8_t +vmd_get_hotplug_bus_numbers(struct vmd_pci_device *dev) +{ + uint8_t bus_number = 0xff; + + if (dev && dev->bus && dev->bus->vmd && + ((dev->bus->vmd->next_bus_number + RESERVED_HOTPLUG_BUSES) < dev->bus->vmd->max_pci_bus)) { + bus_number = RESERVED_HOTPLUG_BUSES; + dev->bus->vmd->next_bus_number += RESERVED_HOTPLUG_BUSES; + } + + return bus_number; +} + +static void +vmd_enable_msix(struct vmd_pci_device *dev) +{ + volatile uint16_t control; + + control = dev->msix_cap->message_control.as_uint16_t | (1 << 14); + dev->msix_cap->message_control.as_uint16_t = control; + control = dev->msix_cap->message_control.as_uint16_t; + dev->msix_cap->message_control.as_uint16_t = (control | (1 << 15)); + control = dev->msix_cap->message_control.as_uint16_t; + control = control & ~(1 << 14); + dev->msix_cap->message_control.as_uint16_t = control; + control = dev->msix_cap->message_control.as_uint16_t; +} + +static void +vmd_disable_msix(struct vmd_pci_device *dev) +{ + volatile uint16_t control; + + control = dev->msix_cap->message_control.as_uint16_t | (1 << 14); + dev->msix_cap->message_control.as_uint16_t = control; + control = dev->msix_cap->message_control.as_uint16_t & ~(1 << 15); + dev->msix_cap->message_control.as_uint16_t = control; + control = dev->msix_cap->message_control.as_uint16_t; +} + +/* + * Set up MSI-X table entries for the port. Vmd MSIX vector 0 is used for + * port interrupt, so vector 0 is mapped to all MSIX entries for the port. + */ +static void +vmd_setup_msix(struct vmd_pci_device *dev, volatile struct pci_msix_table_entry *vmdEntry) +{ + int entry; + + if (!dev || !vmdEntry || !dev->msix_cap) { + return; + } + + vmd_disable_msix(dev); + if (dev->msix_table == NULL || dev->msix_table_size > MAX_MSIX_TABLE_SIZE) { + return; + } + + for (entry = 0; entry < dev->msix_table_size; ++entry) { + dev->msix_table[entry].vector_control = 1; + } + vmd_enable_msix(dev); +} + +static void +vmd_bus_update_bridge_info(struct vmd_pci_device *bridge) +{ + /* Update the subordinate bus of all bridges above this bridge */ + volatile struct vmd_pci_device *dev = bridge; + uint8_t subordinate_bus; + + if (!dev) { + return; + } + subordinate_bus = bridge->header->one.subordinate; + while (dev->parent_bridge != NULL) { + dev = dev->parent_bridge; + if (dev->header->one.subordinate < subordinate_bus) { + dev->header->one.subordinate = subordinate_bus; + subordinate_bus = dev->header->one.subordinate; + } + } +} + +static bool +vmd_is_supported_device(struct vmd_pci_device *dev) +{ + return dev->class == PCI_CLASS_STORAGE_EXPRESS; +} + +static int +vmd_dev_map_bar(struct spdk_pci_device *pci_dev, uint32_t bar, + void **mapped_addr, uint64_t *phys_addr, uint64_t *size) +{ + struct vmd_pci_device *dev = SPDK_CONTAINEROF(pci_dev, struct vmd_pci_device, pci); + + *size = dev->bar[bar].size; + *phys_addr = dev->bar[bar].start; + *mapped_addr = (void *)dev->bar[bar].vaddr; + + return 0; +} + +static int +vmd_dev_unmap_bar(struct spdk_pci_device *_dev, uint32_t bar, void *addr) +{ + return 0; +} + +static int +vmd_dev_cfg_read(struct spdk_pci_device *_dev, void *value, uint32_t len, + uint32_t offset) +{ + struct vmd_pci_device *dev = SPDK_CONTAINEROF(_dev, struct vmd_pci_device, pci); + volatile uint8_t *src = (volatile uint8_t *)dev->header; + uint8_t *dst = value; + size_t i; + + if (len + offset > PCI_MAX_CFG_SIZE) { + return -1; + } + + for (i = 0; i < len; ++i) { + dst[i] = src[offset + i]; + } + + return 0; +} + +static int +vmd_dev_cfg_write(struct spdk_pci_device *_dev, void *value, + uint32_t len, uint32_t offset) +{ + struct vmd_pci_device *dev = SPDK_CONTAINEROF(_dev, struct vmd_pci_device, pci); + volatile uint8_t *dst = (volatile uint8_t *)dev->header; + uint8_t *src = value; + size_t i; + + if ((len + offset) > PCI_MAX_CFG_SIZE) { + return -1; + } + + for (i = 0; i < len; ++i) { + dst[offset + i] = src[i]; + } + + return 0; +} + +static void +vmd_dev_detach(struct spdk_pci_device *dev) +{ + struct vmd_pci_device *vmd_device = (struct vmd_pci_device *)dev; + struct vmd_pci_device *bus_device = vmd_device->bus->self; + struct vmd_pci_bus *bus = vmd_device->bus; + size_t i, num_bars = vmd_device->header_type ? 2 : 6; + + spdk_pci_unhook_device(dev); + TAILQ_REMOVE(&bus->dev_list, vmd_device, tailq); + + /* Release the hotplug region if the device is under hotplug-capable bus */ + if (bus_device && bus_device->hotplug_capable) { + for (i = 0; i < num_bars; ++i) { + if (vmd_device->bar[i].start != 0) { + vmd_hotplug_free_addr(&bus_device->hp, vmd_device->bar[i].start); + } + } + } + + free(dev); +} + +static void +vmd_dev_init(struct vmd_pci_device *dev) +{ + uint8_t bdf[32]; + + dev->pci.addr.domain = dev->bus->vmd->domain; + dev->pci.addr.bus = dev->bus->bus_number; + dev->pci.addr.dev = dev->devfn; + dev->pci.addr.func = 0; + dev->pci.id.vendor_id = dev->header->common.vendor_id; + dev->pci.id.device_id = dev->header->common.device_id; + dev->pci.type = "vmd"; + dev->pci.map_bar = vmd_dev_map_bar; + dev->pci.unmap_bar = vmd_dev_unmap_bar; + dev->pci.cfg_read = vmd_dev_cfg_read; + dev->pci.cfg_write = vmd_dev_cfg_write; + dev->hotplug_capable = false; + if (dev->pcie_cap != NULL) { + dev->cached_slot_control = dev->pcie_cap->slot_control; + } + + if (vmd_is_supported_device(dev)) { + spdk_pci_addr_fmt(bdf, sizeof(bdf), &dev->pci.addr); + SPDK_DEBUGLOG(SPDK_LOG_VMD, "Initalizing NVMe device at %s\n", bdf); + dev->pci.parent = dev->bus->vmd->pci; + spdk_pci_hook_device(spdk_pci_nvme_get_driver(), &dev->pci); + } +} + +/* + * Scans a single bus for all devices attached and return a count of + * how many devices found. In the VMD topology, it is assume there are no multi- + * function devices. Hence a bus(bridge) will not have multi function with both type + * 0 and 1 header. + * + * The other option for implementing this function is the bus is an int and + * create a new device PciBridge. PciBridge would inherit from PciDevice with extra fields, + * sub/pri/sec bus. The input becomes PciPort, bus number and parent_bridge. + * + * The bus number is scanned and if a device is found, based on the header_type, create + * either PciBridge(1) or PciDevice(0). + * + * If a PciBridge, assign bus numbers and rescan new bus. The currenty PciBridge being + * scanned becomes the passed in parent_bridge with the new bus number. + * + * The linked list becomes list of pciBridges with PciDevices attached. + * + * Return count of how many devices found(type1 + type 0 header devices) + */ +static uint8_t +vmd_scan_single_bus(struct vmd_pci_bus *bus, struct vmd_pci_device *parent_bridge) +{ + /* assuming only single function devices are on the bus */ + struct vmd_pci_device *new_dev; + struct vmd_adapter *vmd; + union express_slot_capabilities_register slot_cap; + struct vmd_pci_bus *new_bus; + uint8_t device_number, dev_cnt = 0; + uint8_t new_bus_num; + + for (device_number = 0; device_number < 32; device_number++) { + new_dev = vmd_alloc_dev(bus, device_number); + if (new_dev == NULL) { + continue; + } + + dev_cnt++; + if (new_dev->header->common.header_type & PCI_HEADER_TYPE_BRIDGE) { + slot_cap.as_uint32_t = 0; + if (new_dev->pcie_cap != NULL) { + slot_cap.as_uint32_t = new_dev->pcie_cap->slot_cap.as_uint32_t; + } + + new_bus_num = vmd_get_next_bus_number(bus->vmd->is_hotplug_scan ? new_dev : NULL, bus->vmd); + if (new_bus_num == 0xff) { + free(new_dev); + return dev_cnt; + } + new_bus = vmd_create_new_bus(bus, new_dev, new_bus_num); + if (!new_bus) { + free(new_dev); + return dev_cnt; + } + new_bus->primary_bus = bus->secondary_bus; + new_bus->self = new_dev; + new_dev->bus_object = new_bus; + + if (slot_cap.bit_field.hotplug_capable && new_dev->pcie_cap != NULL && + new_dev->pcie_cap->express_cap_register.bit_field.slot_implemented) { + new_bus->hotplug_buses = vmd_get_hotplug_bus_numbers(new_dev); + new_bus->subordinate_bus += new_bus->hotplug_buses; + + /* Attach hot plug instance if HP is supported */ + /* Hot inserted SSDs can be assigned port bus of sub-ordinate + 1 */ + SPDK_DEBUGLOG(SPDK_LOG_VMD, "hotplug_capable/slot_implemented = " + "%x:%x\n", slot_cap.bit_field.hotplug_capable, + new_dev->pcie_cap->express_cap_register.bit_field.slot_implemented); + } + + new_dev->parent_bridge = parent_bridge; + new_dev->header->one.primary = new_bus->primary_bus; + new_dev->header->one.secondary = new_bus->secondary_bus; + new_dev->header->one.subordinate = new_bus->subordinate_bus; + + vmd_bus_update_bridge_info(new_dev); + TAILQ_INSERT_TAIL(&bus->vmd->bus_list, new_bus, tailq); + + vmd_dev_init(new_dev); + + if (slot_cap.bit_field.hotplug_capable && new_dev->pcie_cap != NULL && + new_dev->pcie_cap->express_cap_register.bit_field.slot_implemented) { + vmd_init_hotplug(new_dev, new_bus); + } + + dev_cnt += vmd_scan_single_bus(new_bus, new_dev); + if (new_dev->pcie_cap != NULL) { + if (new_dev->pcie_cap->express_cap_register.bit_field.device_type == SwitchUpstreamPort) { + return dev_cnt; + } + } + } else { + /* Attach the device to the current bus and assign base addresses */ + TAILQ_INSERT_TAIL(&bus->dev_list, new_dev, tailq); + g_end_device_count++; + if (vmd_assign_base_addrs(new_dev)) { + vmd_setup_msix(new_dev, &bus->vmd->msix_table[0]); + vmd_dev_init(new_dev); + if (vmd_is_supported_device(new_dev)) { + vmd = bus->vmd; + vmd->target[vmd->nvme_count] = new_dev; + vmd->nvme_count++; + } + } else { + SPDK_DEBUGLOG(SPDK_LOG_VMD, "Removing failed device:%p\n", new_dev); + TAILQ_REMOVE(&bus->dev_list, new_dev, tailq); + free(new_dev); + if (dev_cnt) { + dev_cnt--; + } + } + } + } + + return dev_cnt; +} + +static void +vmd_print_pci_info(struct vmd_pci_device *dev) +{ + if (!dev) { + return; + } + + if (dev->pcie_cap != NULL) { + SPDK_INFOLOG(SPDK_LOG_VMD, "PCI DEVICE: [%04X:%04X] type(%x) : %s\n", + dev->header->common.vendor_id, dev->header->common.device_id, + dev->pcie_cap->express_cap_register.bit_field.device_type, + device_type[dev->pcie_cap->express_cap_register.bit_field.device_type]); + } else { + SPDK_INFOLOG(SPDK_LOG_VMD, "PCI DEVICE: [%04X:%04X]\n", + dev->header->common.vendor_id, dev->header->common.device_id); + } + + SPDK_INFOLOG(SPDK_LOG_VMD, "\tDOMAIN:BDF: %04x:%02x:%02x:%x\n", dev->pci.addr.domain, + dev->pci.addr.bus, dev->pci.addr.dev, dev->pci.addr.func); + + if (!(dev->header_type & PCI_HEADER_TYPE_BRIDGE) && dev->bus) { + SPDK_INFOLOG(SPDK_LOG_VMD, "\tbase addr: %x : %p\n", + dev->header->zero.BAR[0], (void *)dev->bar[0].vaddr); + } + + if ((dev->header_type & PCI_HEADER_TYPE_BRIDGE)) { + SPDK_INFOLOG(SPDK_LOG_VMD, "\tPrimary = %d, Secondary = %d, Subordinate = %d\n", + dev->header->one.primary, dev->header->one.secondary, dev->header->one.subordinate); + if (dev->pcie_cap && dev->pcie_cap->express_cap_register.bit_field.slot_implemented) { + SPDK_INFOLOG(SPDK_LOG_VMD, "\tSlot implemented on this device.\n"); + if (dev->pcie_cap->slot_cap.bit_field.hotplug_capable) { + SPDK_INFOLOG(SPDK_LOG_VMD, "Device has HOT-PLUG capable slot.\n"); + } + } + } + + if (dev->sn_cap != NULL) { + uint8_t *snLow = (uint8_t *)&dev->sn_cap->sn_low; + uint8_t *snHi = (uint8_t *)&dev->sn_cap->sn_hi; + + SPDK_INFOLOG(SPDK_LOG_VMD, "\tSN: %02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x\n", + snHi[3], snHi[2], snHi[1], snHi[0], snLow[3], snLow[2], snLow[1], snLow[0]); + } +} + +static void +vmd_cache_scan_info(struct vmd_pci_device *dev) +{ + uint32_t reg __attribute__((unused)); + + if (dev->header_type == PCI_HEADER_TYPE_NORMAL) { + return; + } + + SPDK_DEBUGLOG(SPDK_LOG_VMD, "vendor/device id:%x:%x\n", dev->header->common.vendor_id, + dev->header->common.device_id); + + if (vmd_device_is_root_port(dev)) { + dev->header->one.prefetch_base_upper = VMD_UPPER_BASE_SIGNATURE; + reg = dev->header->one.prefetch_base_upper; + dev->header->one.prefetch_limit_upper = VMD_UPPER_LIMIT_SIGNATURE; + reg = dev->header->one.prefetch_limit_upper; + + SPDK_DEBUGLOG(SPDK_LOG_VMD, "prefetch: %x:%x\n", + dev->header->one.prefetch_base_upper, + dev->header->one.prefetch_limit_upper); + } +} + +static uint8_t +vmd_scan_pcibus(struct vmd_pci_bus *bus) +{ + struct vmd_pci_bus *bus_entry; + struct vmd_pci_device *dev; + uint8_t dev_cnt; + + g_end_device_count = 0; + TAILQ_INSERT_TAIL(&bus->vmd->bus_list, bus, tailq); + bus->vmd->next_bus_number = bus->bus_number + 1; + dev_cnt = vmd_scan_single_bus(bus, NULL); + + SPDK_DEBUGLOG(SPDK_LOG_VMD, "VMD scan found %u devices\n", dev_cnt); + SPDK_DEBUGLOG(SPDK_LOG_VMD, "VMD scan found %u END DEVICES\n", g_end_device_count); + + SPDK_INFOLOG(SPDK_LOG_VMD, "PCIe devices attached to VMD %04x:%02x:%02x:%x...\n", + bus->vmd->pci->addr.domain, bus->vmd->pci->addr.bus, + bus->vmd->pci->addr.dev, bus->vmd->pci->addr.func); + + TAILQ_FOREACH(bus_entry, &bus->vmd->bus_list, tailq) { + if (bus_entry->self != NULL) { + vmd_print_pci_info(bus_entry->self); + vmd_cache_scan_info(bus_entry->self); + } + + TAILQ_FOREACH(dev, &bus_entry->dev_list, tailq) { + vmd_print_pci_info(dev); + } + } + + return dev_cnt; +} + +static int +vmd_map_bars(struct vmd_adapter *vmd, struct spdk_pci_device *dev) +{ + int rc; + + rc = spdk_pci_device_map_bar(dev, 0, (void **)&vmd->cfg_vaddr, + &vmd->cfgbar, &vmd->cfgbar_size); + if (rc == 0) { + rc = spdk_pci_device_map_bar(dev, 2, (void **)&vmd->mem_vaddr, + &vmd->membar, &vmd->membar_size); + } + + if (rc == 0) { + rc = spdk_pci_device_map_bar(dev, 4, (void **)&vmd->msix_vaddr, + &vmd->msixbar, &vmd->msixbar_size); + } + + if (rc == 0) { + vmd->physical_addr = vmd->membar; + vmd->current_addr_size = vmd->membar_size; + } + return rc; +} + +static int +vmd_enumerate_devices(struct vmd_adapter *vmd) +{ + vmd->vmd_bus.vmd = vmd; + vmd->vmd_bus.secondary_bus = vmd->vmd_bus.subordinate_bus = 0; + vmd->vmd_bus.primary_bus = vmd->vmd_bus.bus_number = 0; + vmd->vmd_bus.domain = vmd->pci->addr.domain; + + return vmd_scan_pcibus(&vmd->vmd_bus); +} + +struct vmd_pci_device * +vmd_find_device(const struct spdk_pci_addr *addr) +{ + struct vmd_pci_bus *bus; + struct vmd_pci_device *dev; + int i; + + for (i = 0; i < MAX_VMD_TARGET; ++i) { + TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) { + if (bus->self) { + if (spdk_pci_addr_compare(&bus->self->pci.addr, addr) == 0) { + return bus->self; + } + } + + TAILQ_FOREACH(dev, &bus->dev_list, tailq) { + if (spdk_pci_addr_compare(&dev->pci.addr, addr) == 0) { + return dev; + } + } + } + } + + return NULL; +} + +static int +vmd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) +{ + uint32_t cmd_reg = 0; + char bdf[32] = {0}; + struct vmd_container *vmd_c = ctx; + size_t i; + + spdk_pci_device_cfg_read32(pci_dev, &cmd_reg, 4); + cmd_reg |= 0x6; /* PCI bus master/memory enable. */ + spdk_pci_device_cfg_write32(pci_dev, cmd_reg, 4); + + spdk_pci_addr_fmt(bdf, sizeof(bdf), &pci_dev->addr); + SPDK_DEBUGLOG(SPDK_LOG_VMD, "Found a VMD[ %d ] at %s\n", vmd_c->count, bdf); + + /* map vmd bars */ + i = vmd_c->count; + vmd_c->vmd[i].pci = pci_dev; + vmd_c->vmd[i].vmd_index = i; + vmd_c->vmd[i].domain = + (pci_dev->addr.bus << 16) | (pci_dev->addr.dev << 8) | pci_dev->addr.func; + vmd_c->vmd[i].max_pci_bus = PCI_MAX_BUS_NUMBER; + TAILQ_INIT(&vmd_c->vmd[i].bus_list); + + if (vmd_map_bars(&vmd_c->vmd[i], pci_dev) == -1) { + return -1; + } + + SPDK_DEBUGLOG(SPDK_LOG_VMD, "vmd config bar(%p) vaddr(%p) size(%x)\n", + (void *)vmd_c->vmd[i].cfgbar, (void *)vmd_c->vmd[i].cfg_vaddr, + (uint32_t)vmd_c->vmd[i].cfgbar_size); + SPDK_DEBUGLOG(SPDK_LOG_VMD, "vmd mem bar(%p) vaddr(%p) size(%x)\n", + (void *)vmd_c->vmd[i].membar, (void *)vmd_c->vmd[i].mem_vaddr, + (uint32_t)vmd_c->vmd[i].membar_size); + SPDK_DEBUGLOG(SPDK_LOG_VMD, "vmd msix bar(%p) vaddr(%p) size(%x)\n\n", + (void *)vmd_c->vmd[i].msixbar, (void *)vmd_c->vmd[i].msix_vaddr, + (uint32_t)vmd_c->vmd[i].msixbar_size); + + vmd_c->count = i + 1; + + vmd_enumerate_devices(&vmd_c->vmd[i]); + + return 0; +} + +int +spdk_vmd_pci_device_list(struct spdk_pci_addr vmd_addr, struct spdk_pci_device *nvme_list) +{ + int cnt = 0; + struct vmd_pci_bus *bus; + struct vmd_pci_device *dev; + + if (!nvme_list) { + return -1; + } + + for (int i = 0; i < MAX_VMD_TARGET; ++i) { + if (spdk_pci_addr_compare(&vmd_addr, &g_vmd_container.vmd[i].pci->addr) == 0) { + TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) { + TAILQ_FOREACH(dev, &bus->dev_list, tailq) { + nvme_list[cnt++] = dev->pci; + if (!dev->is_hooked) { + vmd_dev_init(dev); + dev->is_hooked = 1; + } + } + } + } + } + + return cnt; +} + +static void +vmd_clear_hotplug_status(struct vmd_pci_bus *bus) +{ + struct vmd_pci_device *device = bus->self; + uint16_t status __attribute__((unused)); + + status = device->pcie_cap->slot_status.as_uint16_t; + device->pcie_cap->slot_status.as_uint16_t = status; + status = device->pcie_cap->slot_status.as_uint16_t; + + status = device->pcie_cap->link_status.as_uint16_t; + device->pcie_cap->link_status.as_uint16_t = status; + status = device->pcie_cap->link_status.as_uint16_t; +} + +static void +vmd_bus_handle_hotplug(struct vmd_pci_bus *bus) +{ + uint8_t num_devices, sleep_count; + + for (sleep_count = 0; sleep_count < 20; ++sleep_count) { + /* Scan until a new device is found */ + num_devices = vmd_scan_single_bus(bus, bus->self); + if (num_devices > 0) { + break; + } + + spdk_delay_us(200000); + } + + if (num_devices == 0) { + SPDK_ERRLOG("Timed out while scanning for hotplugged devices\n"); + } +} + +static void +vmd_bus_handle_hotremove(struct vmd_pci_bus *bus) +{ + struct vmd_pci_device *device, *tmpdev; + + TAILQ_FOREACH_SAFE(device, &bus->dev_list, tailq, tmpdev) { + if (!vmd_bus_device_present(bus, device->devfn)) { + device->pci.internal.pending_removal = true; + + /* If the device isn't attached, remove it immediately */ + if (!device->pci.internal.attached) { + vmd_dev_detach(&device->pci); + } + } + } +} + +int +spdk_vmd_hotplug_monitor(void) +{ + struct vmd_pci_bus *bus; + struct vmd_pci_device *device; + int num_hotplugs = 0; + uint32_t i; + + for (i = 0; i < g_vmd_container.count; ++i) { + TAILQ_FOREACH(bus, &g_vmd_container.vmd[i].bus_list, tailq) { + device = bus->self; + if (device == NULL || !device->hotplug_capable) { + continue; + } + + if (device->pcie_cap->slot_status.bit_field.datalink_state_changed != 1) { + continue; + } + + if (device->pcie_cap->link_status.bit_field.datalink_layer_active == 1) { + SPDK_DEBUGLOG(SPDK_LOG_VMD, "Device hotplug detected on bus " + "%"PRIu32"\n", bus->bus_number); + vmd_bus_handle_hotplug(bus); + } else { + SPDK_DEBUGLOG(SPDK_LOG_VMD, "Device hotremove detected on bus " + "%"PRIu32"\n", bus->bus_number); + vmd_bus_handle_hotremove(bus); + } + + vmd_clear_hotplug_status(bus); + num_hotplugs++; + } + } + + return num_hotplugs; +} + +int +spdk_vmd_init(void) +{ + return spdk_pci_enumerate(spdk_pci_vmd_get_driver(), vmd_enum_cb, &g_vmd_container); +} + +void +spdk_vmd_fini(void) +{ + uint32_t i; + + for (i = 0; i < g_vmd_container.count; ++i) { + spdk_pci_device_detach(g_vmd_container.vmd[i].pci); + } +} + +SPDK_LOG_REGISTER_COMPONENT("vmd", SPDK_LOG_VMD) diff --git a/src/spdk/lib/vmd/vmd.h b/src/spdk/lib/vmd/vmd.h new file mode 100644 index 000000000..46490a6f7 --- /dev/null +++ b/src/spdk/lib/vmd/vmd.h @@ -0,0 +1,201 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef VMD_H +#define VMD_H + +#include "spdk/stdinc.h" +#include "spdk/vmd.h" +#include "spdk/env.h" +#include "spdk/util.h" +#include "spdk_internal/log.h" +#include "vmd_spec.h" + +struct vmd_hot_plug; +struct vmd_adapter; +struct vmd_pci_device; + +struct pci_bars { + uint64_t vaddr; + uint64_t start; + uint32_t size; +}; + +struct vmd_pci_bus { + struct vmd_adapter *vmd; + struct vmd_pci_bus *parent; /* parent bus that this bus is attached to(primary bus. */ + struct vmd_pci_device *self; /* Pci device that describes this bus(bar, bus numbers, etc */ + + uint32_t domain : 8; + uint32_t hotplug_buses : 10; + uint32_t is_added : 1; + uint32_t hp_event_queued : 1; + uint32_t rsv : 12; + + uint32_t bus_number : 8; + uint32_t primary_bus : 8; + uint32_t secondary_bus : 8; + uint32_t subordinate_bus : 8; + + TAILQ_HEAD(, vmd_pci_device) dev_list; /* list of pci end device attached to this bus */ + TAILQ_ENTRY(vmd_pci_bus) tailq; /* link for all buses found during scan */ +}; + +/* + * memory element for base address assignment and reuse + */ +struct pci_mem_mgr { + uint32_t size : 30; /* size of memory element */ + uint32_t in_use : 1; + uint32_t rsv : 1; + uint64_t addr; + TAILQ_ENTRY(pci_mem_mgr) tailq; +}; + +struct vmd_hot_plug { + uint32_t count : 12; + uint32_t reserved_bus_count : 4; + uint32_t max_hotplug_bus_number : 8; + uint32_t next_bus_number : 8; + struct pci_bars bar; + union express_slot_status_register slot_status; + struct pci_mem_mgr mem[ADDR_ELEM_COUNT]; + uint8_t bus_numbers[RESERVED_HOTPLUG_BUSES]; + struct vmd_pci_bus *bus; + TAILQ_HEAD(, pci_mem_mgr) free_mem_queue; + TAILQ_HEAD(, pci_mem_mgr) alloc_mem_queue; + TAILQ_HEAD(, pci_mem_mgr) unused_mem_queue; +}; + +struct vmd_pci_device { + struct spdk_pci_device pci; + struct pci_bars bar[6]; + + struct vmd_pci_device *parent_bridge; + struct vmd_pci_bus *bus, *parent; + struct vmd_pci_bus *bus_object; /* bus tracks pci bus associated with this dev if type 1 dev. */ + struct vmd_pci_bus *subordinate; + volatile struct pci_header *header; + volatile struct pci_express_cap *pcie_cap; + volatile struct pci_msix_capability *msix_cap; + volatile struct pci_msi_cap *msi_cap; + volatile struct serial_number_capability *sn_cap; + volatile struct pci_msix_table_entry *msix_table; + + TAILQ_ENTRY(vmd_pci_device) tailq; + + uint32_t class; + uint16_t vid; + uint16_t did; + uint16_t pcie_flags, msix_table_size; + uint32_t devfn; + bool hotplug_capable; + + uint32_t header_type : 1; + uint32_t multifunction : 1; + uint32_t hotplug_bridge : 1; + uint32_t is_added : 1; + uint32_t is_hooked : 1; + uint32_t rsv1 : 12; + uint32_t target : 16; + + struct vmd_hot_plug hp; + /* Cached version of the slot_control register */ + union express_slot_control_register cached_slot_control; +}; + +/* + * The VMD adapter + */ +struct vmd_adapter { + struct spdk_pci_device *pci; + uint32_t domain; + /* physical and virtual VMD bars */ + uint64_t cfgbar, cfgbar_size; + uint64_t membar, membar_size; + uint64_t msixbar, msixbar_size; + volatile uint8_t *cfg_vaddr; + volatile uint8_t *mem_vaddr; + volatile uint8_t *msix_vaddr; + volatile struct pci_msix_table_entry *msix_table; + uint32_t bar_sizes[6]; + + uint64_t physical_addr; + uint32_t current_addr_size; + + uint32_t next_bus_number : 10; + uint32_t max_pci_bus : 10; + uint32_t is_hotplug_scan : 1; + uint32_t is_ready : 1; + uint32_t processing_hp : 1; + uint32_t max_payload_size: 3; + uint32_t root_port_updated : 1; + uint32_t scan_completed : 1; + uint32_t rsv : 4; + + /* end devices attached to vmd adapters */ + struct vmd_pci_device *target[MAX_VMD_TARGET]; + uint32_t dev_count : 16; + uint32_t nvme_count : 8; + uint32_t vmd_index : 8; + + struct vmd_pci_bus vmd_bus; + + TAILQ_HEAD(, vmd_pci_bus) bus_list; + + struct event_fifo *hp_queue; +}; + +/* TODO: Temporary stubs for Hot Plug interface */ +static inline struct vmd_pci_bus * +vmd_is_dev_in_hotplug_path(struct vmd_pci_device *dev) +{ + return NULL; +} + +static inline void +vmd_hp_enable_hotplug(struct vmd_hot_plug *hp) +{ + +} + +static inline uint8_t +vmd_hp_get_next_bus_number(struct vmd_hot_plug *hp) +{ + assert(false); + return 0; +} + +struct vmd_pci_device *vmd_find_device(const struct spdk_pci_addr *addr); + +#endif /* VMD_H */ diff --git a/src/spdk/lib/vmd/vmd_spec.h b/src/spdk/lib/vmd/vmd_spec.h new file mode 100644 index 000000000..07a4a113d --- /dev/null +++ b/src/spdk/lib/vmd/vmd_spec.h @@ -0,0 +1,473 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef VMD_SPEC_H +#define VMD_SPEC_H + +#define MAX_VMD_SUPPORTED 48 /* max number of vmd controllers in a system - */ +#define VMD_DOMAIN_START 0x201D + +#define PCI_INVALID_VENDORID 0xFFFF +#define ONE_MB (1<<20) +#define PCI_OFFSET_OF(object, member) ((uint32_t)&((object*)0)->member) +#define TWOS_COMPLEMENT(value) (~(value) + 1) + +#define VMD_UPPER_BASE_SIGNATURE 0xFFFFFFEF +#define VMD_UPPER_LIMIT_SIGNATURE 0xFFFFFFED + +/* + * BAR assignment constants + */ +#define PCI_DWORD_SHIFT 32 +#define PCI_BASE_ADDR_MASK 0xFFFFFFF0 +#define PCI_BAR_MEMORY_MASK 0x0000000F +#define PCI_BAR_MEMORY_MEM_IND 0x1 +#define PCI_BAR_MEMORY_TYPE 0x6 +#define PCI_BAR_MEMORY_PREFETCH 0x8 +#define PCI_BAR_MEMORY_TYPE_32 0x0 +#define PCI_BAR_MEMORY_TYPE_64 0x4 +#define PCI_BAR_MB_MASK 0xFFFFF +#define PCI_PCI_BRIDGE_ADDR_DEF 0xFFF0 +#define PCI_BRIDGE_MEMORY_MASK 0xFFF0 +#define PCI_BRIDGE_PREFETCH_64 0x0001 +#define PCI_BRIDGE_MEMORY_SHIFT 16 +#define PCI_CONFIG_ACCESS_DELAY 500 + +#define PCI_MAX_CFG_SIZE 0x1000 + +#define PCI_HEADER_TYPE 0x0e +#define PCI_HEADER_TYPE_NORMAL 0 +#define PCI_HEADER_TYPE_BRIDGE 1 +#define PCI_MULTI_FUNCTION 0x80 + +#define PCI_COMMAND_MEMORY 0x2 +#define PCI_COMMAND_MASTER 0x4 + +#define PCIE_TYPE_FLAGS 0xf0 +#define PCIE_TYPE_SHIFT 4 +#define PCIE_TYPE_ROOT_PORT 0x4 +#define PCIE_TYPE_DOWNSTREAM 0x6 + +#define PCI_CLASS_STORAGE_EXPRESS 0x010802 +#define ADDR_ELEM_COUNT 32 +#define PCI_MAX_BUS_NUMBER 0x7F +#define RESERVED_HOTPLUG_BUSES 1 +#define isHotPlugCapable(slotCap) ((slotCap) & (1<<6)) +#define CONFIG_OFFSET_ADDR(bus, device, function, reg) (((bus)<<20) | (device)<<15 | (function<<12) | (reg)) +#define BRIDGE_BASEREG(reg) (0xFFF0 & ((reg)>>16)) + +#define MISCCTRLSTS_0_OFFSET 0x188 +#define ENABLE_ACPI_MODE_FOR_HOTPLUG (1 << 3) + +/* Bit encodings for Command Register */ +#define IO_SPACE_ENABLE 0x0001 +#define MEMORY_SPACE_ENABLE 0x0002 +#define BUS_MASTER_ENABLE 0x0004 + +/* Bit encodings for Status Register */ +#define PCI_CAPABILITIES_LIST 0x0010 +#define PCI_RECEIVED_TARGET_ABORT 0x1000 +#define PCI_RECEIVED_MASTER_ABORT 0x2000 +#define PCI_SIGNALED_SYSTEM_ERROR 0x4000 +#define PCI_DETECTED_PARITY_ERROR 0x8000 + +/* Capability IDs */ +#define CAPABILITY_ID_POWER_MANAGEMENT 0x01 +#define CAPABILITY_ID_MSI 0x05 +#define CAPABILITY_ID_PCI_EXPRESS 0x10 +#define CAPABILITY_ID_MSIX 0x11 + +#define PCI_MSIX_ENABLE (1 << 15) /* bit 15 of MSIX Message Control */ +#define PCI_MSIX_FUNCTION_MASK (1 << 14) /* bit 14 of MSIX Message Control */ + +/* extended capability */ +#define EXTENDED_CAPABILITY_OFFSET 0x100 +#define DEVICE_SERIAL_NUMBER_CAP_ID 0x3 + +#define BAR_SIZE (1 << 20) + +struct pci_enhanced_capability_header { + uint16_t capability_id; + uint16_t version: 4; + uint16_t next: 12; +}; + +struct serial_number_capability { + struct pci_enhanced_capability_header hdr; + uint32_t sn_low; + uint32_t sn_hi; +}; + +struct pci_header_common { + uint16_t vendor_id; + uint16_t device_id; + uint16_t command; + uint16_t status; + uint32_t rev_class; + uint8_t cache_line_size; + uint8_t master_lat_timer; + uint8_t header_type; + uint8_t BIST; + uint8_t rsvd12[36]; + uint8_t cap_pointer; + uint8_t rsvd53[7]; + uint8_t int_line; + uint8_t int_pin; + uint8_t rsvd62[2]; +}; + +struct pci_header_zero { + uint16_t vendor_id; + uint16_t device_id; + uint16_t command; + uint16_t status; + uint32_t rev_class; + uint8_t cache_line_size; + uint8_t master_lat_timer; + uint8_t header_type; + uint8_t BIST; + uint32_t BAR[6]; + uint32_t carbus_cis_pointer; + uint16_t ssvid; + uint16_t ssid; + uint32_t exp_rom_base_addr; + uint8_t cap_pointer; + uint8_t rsvd53[7]; + uint8_t intLine; + uint8_t int_pin; + uint8_t min_gnt; + uint8_t max_lat; +}; + +struct pci_header_one { + uint16_t vendor_id; + uint16_t device_id; + uint16_t command; + uint16_t status; + uint32_t rev_class; + uint8_t cache_line_size; + uint8_t master_lat_timer; + uint8_t header_type; + uint8_t BIST; + uint32_t BAR[2]; + uint8_t primary; + uint8_t secondary; + uint8_t subordinate; + uint8_t secondary_lat_timer; + uint8_t io_base; + uint8_t io_limit; + uint16_t secondary_status; + uint16_t mem_base; + uint16_t mem_limit; + uint16_t prefetch_base; + uint16_t prefetch_limit; + uint32_t prefetch_base_upper; + uint32_t prefetch_limit_upper; + uint16_t io_base_upper; + uint16_t io_limit_upper; + uint8_t cap_pointer; + uint8_t rsvd53[3]; + uint32_t exp_romBase_addr; + uint8_t int_line; + uint8_t int_pin; + uint16_t bridge_control; +}; + +struct pci_capabilities_header { + uint8_t capability_id; + uint8_t next; +}; + +/* + * MSI capability structure for msi interrupt vectors + */ +#define MAX_MSIX_TABLE_SIZE 0x800 +#define MSIX_ENTRY_VECTOR_CTRL_MASKBIT 1 +#define PORT_INT_VECTOR 0; +#define CLEAR_MSIX_DESTINATION_ID 0xfff00fff +struct pci_msi_cap { + struct pci_capabilities_header header; + union _MsiControl { + uint16_t as_uint16_t; + struct _PCI_MSI_MESSAGE_CONTROL { + uint16_t msi_enable : 1; + uint16_t multiple_message_capable : 3; + uint16_t multiple_message_enable : 3; + uint16_t capable_of_64bits : 1; + uint16_t per_vector_mask_capable : 1; + uint16_t reserved : 7; + } bit; + } message_control; + union { + struct _PCI_MSI_MESSAGE_ADDRESS { + uint32_t reserved : 2; + uint32_t address : 30; + } reg; + uint32_t raw; + } message_address_lower; + union { + struct _Option32_bit { + uint16_t message_data; + } option32_bit; + struct _Option64_bit { + uint32_t message_address_upper; + uint16_t message_data; + uint16_t reserved; + uint32_t mask_bits; + uint32_t pending_bits; + } option64_bit; + }; +}; + +struct pcix_table_pointer { + union { + struct { + uint32_t BaseIndexRegister : 3; + uint32_t Reserved : 29; + } TableBIR; + uint32_t TableOffset; + }; +}; + +struct pci_msix_capability { + struct pci_capabilities_header header; + union _MsixControl { + uint16_t as_uint16_t; + struct msg_ctrl { + uint16_t table_size : 11; + uint16_t reserved : 3; + uint16_t function_mask : 1; + uint16_t msix_enable : 1; + } bit; + } message_control; + + struct pcix_table_pointer message_table; + struct pcix_table_pointer pba_table; +}; + +struct pci_msix_table_entry { + volatile uint32_t message_addr_lo; + volatile uint32_t message_addr_hi; + volatile uint32_t message_data; + volatile uint32_t vector_control; +}; + +/* + * Pci express capability + */ +enum PciExpressCapabilities { + /* 0001b Legacy PCI Express Endpoint */ + LegacyEndpoint = 0x1, + /* 0000b PCI Express Endpoint */ + ExpressEndpoint = 0x0, + /* 0100b Root Port of PCI Express Root Complex* */ + RootComplexRootPort = 0x4, + /* 0101b Upstream Port of PCI Express Switch* */ + SwitchUpstreamPort = 0x5, + /* 0110b Downstream Port of PCI Express Switch* */ + SwitchDownStreamPort = 0x6, + /* 0111b PCI Express to PCI/PCI-X Bridge* */ + ExpressToPciBridge = 0x7, + /* 1000b PCI/PCI-X to PCI Express Bridge* */ + PciToExpressBridge = 0x8, + /* 1001b Root Complex Integrated Endpoint */ + RCIntegratedEndpoint = 0x9, + /* 1010b Root Complex Event Collector */ + RootComplexEventCollector = 0xa, + InvalidCapability = 0xff +}; + +union express_capability_register { + struct { + uint16_t capability_version : 4; + uint16_t device_type : 4; + uint16_t slot_implemented : 1; + uint16_t interrupt_message_number : 5; + uint16_t rsv : 2; + } bit_field; + uint16_t as_uint16_t; +}; + +union express_slot_capabilities_register { + struct { + uint32_t attention_button_present : 1; + uint32_t power_controller_present : 1; + uint32_t MRL_sensor_present : 1; + uint32_t attention_indicator_present : 1; + uint32_t power_indicator_present : 1; + uint32_t hotplug_surprise : 1; + uint32_t hotplug_capable : 1; + uint32_t slot_power_limit : 8; + uint32_t slotPower_limit_scale : 2; + uint32_t electromechanical_lock_present : 1; + uint32_t no_command_completed_support : 1; + uint32_t physical_slot_number : 13; + } bit_field; + uint32_t as_uint32_t; +}; + +union express_slot_control_register { + struct { + uint16_t attention_button_enable : 1; + uint16_t power_fault_detect_enable : 1; + uint16_t MRLsensor_enable : 1; + uint16_t presence_detect_enable : 1; + uint16_t command_completed_enable : 1; + uint16_t hotplug_interrupt_enable : 1; + uint16_t attention_indicator_control : 2; + uint16_t power_indicator_control : 2; + uint16_t power_controller_control : 1; + uint16_t electromechanical_lockcontrol : 1; + uint16_t datalink_state_change_enable : 1; + uint16_t Rsvd : 3; + } bit_field; + uint16_t as_uint16_t; +}; + +union express_slot_status_register { + struct { + uint16_t attention_button_pressed : 1; + uint16_t power_fault_detected : 1; + uint16_t MRL_sensor_changed : 1; + uint16_t presence_detect_changed : 1; + uint16_t command_completed : 1; + uint16_t MRL_sensor_state : 1; + uint16_t presence_detect_state : 1; + uint16_t electromechanical_lock_engaged : 1; + uint16_t datalink_state_changed : 1; + uint16_t rsvd : 7; + } bit_field; + uint16_t as_uint16_t; +}; + +union express_root_control_register { + struct { + uint16_t CorrectableSerrEnable : 1; + uint16_t NonFatalSerrEnable : 1; + uint16_t FatalSerrEnable : 1; + uint16_t PMEInterruptEnable : 1; + uint16_t CRSSoftwareVisibilityEnable : 1; + uint16_t Rsvd : 11; + } bit_field; + uint16_t as_uint16_t; +}; + +union express_link_capability_register { + struct { + uint32_t maximum_link_speed : 4; + uint32_t maximum_link_width : 6; + uint32_t active_state_pms_support : 2; + uint32_t l0_exit_latency : 3; + uint32_t l1_exit_latency : 3; + uint32_t clock_power_management : 1; + uint32_t surprise_down_error_reporting_capable : 1; + uint32_t datalink_layer_active_reporting_capable : 1; + uint32_t link_bandwidth_notification_capability : 1; + uint32_t aspm_optionality_compliance : 1; + uint32_t rsvd : 1; + uint32_t port_number : 8; + } bit_field; + uint32_t as_uint32_t; +}; + +union express_link_control_register { + struct { + uint16_t active_state_pm_control : 2; + uint16_t rsvd1 : 1; + uint16_t read_completion_boundary : 1; + uint16_t link_disable : 1; + uint16_t retrain_link : 1; + uint16_t common_clock_config : 1; + uint16_t extended_synch : 1; + uint16_t enable_clock_power_management : 1; + uint16_t rsvd2 : 7; + } bit_field; + uint16_t as_uint16_t; +}; + +union express_link_status_register { + struct { + uint16_t link_speed : 4; + uint16_t link_width : 6; + uint16_t undefined : 1; + uint16_t link_training : 1; + uint16_t slot_clock_config : 1; + uint16_t datalink_layer_active : 1; + uint16_t asvd : 2; + } bit_field; + uint16_t as_uint16_t; +}; + +struct pci_express_cap { + uint8_t capid; + uint8_t next_cap; + union express_capability_register express_cap_register; + uint32_t device_cap; + uint16_t device_control; + uint16_t device_status; + union express_link_capability_register link_cap; + union express_link_control_register link_control; + union express_link_status_register link_status; + union express_slot_capabilities_register slot_cap; + union express_slot_control_register slot_control; + union express_slot_status_register slot_status; + uint32_t root_status; + uint32_t deviceCap2; + uint16_t deviceControl2; + uint16_t deviceStatus2; + uint32_t linkCap2; + uint16_t linkControl2; + uint16_t linkStatus2; + uint32_t slotCap2; + uint16_t slotControl2; + uint16_t slotStatus2; +}; + +struct pci_msix_cap { + uint8_t cap_idd; + uint8_t next_cap; + uint16_t msg_control_reg; + uint32_t msix_table_offset; + uint32_t pba_offset; +}; + +struct pci_header { + union { + struct pci_header_common common; + struct pci_header_zero zero; + struct pci_header_one one; + }; +}; + +#endif /* VMD_SPEC_H */ |