summaryrefslogtreecommitdiffstats
path: root/drivers/vfio/pci
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/vfio/pci')
-rw-r--r--drivers/vfio/pci/Kconfig2
-rw-r--r--drivers/vfio/pci/Makefile2
-rw-r--r--drivers/vfio/pci/pds/dirty.c309
-rw-r--r--drivers/vfio/pci/pds/dirty.h18
-rw-r--r--drivers/vfio/pci/vfio_pci_rdwr.c57
-rw-r--r--drivers/vfio/pci/virtio/Kconfig15
-rw-r--r--drivers/vfio/pci/virtio/Makefile3
-rw-r--r--drivers/vfio/pci/virtio/main.c576
8 files changed, 832 insertions, 150 deletions
diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig
index 8125e5f378..18c397df56 100644
--- a/drivers/vfio/pci/Kconfig
+++ b/drivers/vfio/pci/Kconfig
@@ -65,4 +65,6 @@ source "drivers/vfio/pci/hisilicon/Kconfig"
source "drivers/vfio/pci/pds/Kconfig"
+source "drivers/vfio/pci/virtio/Kconfig"
+
endmenu
diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile
index 45167be462..046139a4ec 100644
--- a/drivers/vfio/pci/Makefile
+++ b/drivers/vfio/pci/Makefile
@@ -13,3 +13,5 @@ obj-$(CONFIG_MLX5_VFIO_PCI) += mlx5/
obj-$(CONFIG_HISI_ACC_VFIO_PCI) += hisilicon/
obj-$(CONFIG_PDS_VFIO_PCI) += pds/
+
+obj-$(CONFIG_VIRTIO_VFIO_PCI) += virtio/
diff --git a/drivers/vfio/pci/pds/dirty.c b/drivers/vfio/pci/pds/dirty.c
index 27607d7b90..8ddf4346fc 100644
--- a/drivers/vfio/pci/pds/dirty.c
+++ b/drivers/vfio/pci/pds/dirty.c
@@ -70,7 +70,7 @@ out_free_region_info:
kfree(region_info);
}
-static int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_dirty *dirty,
+static int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_region *region,
unsigned long bytes)
{
unsigned long *host_seq_bmp, *host_ack_bmp;
@@ -85,47 +85,63 @@ static int pds_vfio_dirty_alloc_bitmaps(struct pds_vfio_dirty *dirty,
return -ENOMEM;
}
- dirty->host_seq.bmp = host_seq_bmp;
- dirty->host_ack.bmp = host_ack_bmp;
+ region->host_seq = host_seq_bmp;
+ region->host_ack = host_ack_bmp;
+ region->bmp_bytes = bytes;
return 0;
}
static void pds_vfio_dirty_free_bitmaps(struct pds_vfio_dirty *dirty)
{
- vfree(dirty->host_seq.bmp);
- vfree(dirty->host_ack.bmp);
- dirty->host_seq.bmp = NULL;
- dirty->host_ack.bmp = NULL;
+ if (!dirty->regions)
+ return;
+
+ for (int i = 0; i < dirty->num_regions; i++) {
+ struct pds_vfio_region *region = &dirty->regions[i];
+
+ vfree(region->host_seq);
+ vfree(region->host_ack);
+ region->host_seq = NULL;
+ region->host_ack = NULL;
+ region->bmp_bytes = 0;
+ }
}
static void __pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device *pds_vfio,
- struct pds_vfio_bmp_info *bmp_info)
+ struct pds_vfio_region *region)
{
struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
struct device *pdsc_dev = &pci_physfn(pdev)->dev;
- dma_unmap_single(pdsc_dev, bmp_info->sgl_addr,
- bmp_info->num_sge * sizeof(struct pds_lm_sg_elem),
+ dma_unmap_single(pdsc_dev, region->sgl_addr,
+ region->num_sge * sizeof(struct pds_lm_sg_elem),
DMA_BIDIRECTIONAL);
- kfree(bmp_info->sgl);
+ kfree(region->sgl);
- bmp_info->num_sge = 0;
- bmp_info->sgl = NULL;
- bmp_info->sgl_addr = 0;
+ region->num_sge = 0;
+ region->sgl = NULL;
+ region->sgl_addr = 0;
}
static void pds_vfio_dirty_free_sgl(struct pds_vfio_pci_device *pds_vfio)
{
- if (pds_vfio->dirty.host_seq.sgl)
- __pds_vfio_dirty_free_sgl(pds_vfio, &pds_vfio->dirty.host_seq);
- if (pds_vfio->dirty.host_ack.sgl)
- __pds_vfio_dirty_free_sgl(pds_vfio, &pds_vfio->dirty.host_ack);
+ struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
+
+ if (!dirty->regions)
+ return;
+
+ for (int i = 0; i < dirty->num_regions; i++) {
+ struct pds_vfio_region *region = &dirty->regions[i];
+
+ if (region->sgl)
+ __pds_vfio_dirty_free_sgl(pds_vfio, region);
+ }
}
-static int __pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device *pds_vfio,
- struct pds_vfio_bmp_info *bmp_info,
- u32 page_count)
+static int pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device *pds_vfio,
+ struct pds_vfio_region *region,
+ u32 page_count)
{
struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
struct device *pdsc_dev = &pci_physfn(pdev)->dev;
@@ -147,32 +163,81 @@ static int __pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device *pds_vfio,
return -EIO;
}
- bmp_info->sgl = sgl;
- bmp_info->num_sge = max_sge;
- bmp_info->sgl_addr = sgl_addr;
+ region->sgl = sgl;
+ region->num_sge = max_sge;
+ region->sgl_addr = sgl_addr;
return 0;
}
-static int pds_vfio_dirty_alloc_sgl(struct pds_vfio_pci_device *pds_vfio,
- u32 page_count)
+static void pds_vfio_dirty_free_regions(struct pds_vfio_dirty *dirty)
{
+ vfree(dirty->regions);
+ dirty->regions = NULL;
+ dirty->num_regions = 0;
+}
+
+static int pds_vfio_dirty_alloc_regions(struct pds_vfio_pci_device *pds_vfio,
+ struct pds_lm_dirty_region_info *region_info,
+ u64 region_page_size, u8 num_regions)
+{
+ struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
+ u32 dev_bmp_offset_byte = 0;
int err;
- err = __pds_vfio_dirty_alloc_sgl(pds_vfio, &dirty->host_seq,
- page_count);
- if (err)
- return err;
+ dirty->regions = vcalloc(num_regions, sizeof(struct pds_vfio_region));
+ if (!dirty->regions)
+ return -ENOMEM;
+ dirty->num_regions = num_regions;
+
+ for (int i = 0; i < num_regions; i++) {
+ struct pds_lm_dirty_region_info *ri = &region_info[i];
+ struct pds_vfio_region *region = &dirty->regions[i];
+ u64 region_size, region_start;
+ u32 page_count;
+
+ /* page_count might be adjusted by the device */
+ page_count = le32_to_cpu(ri->page_count);
+ region_start = le64_to_cpu(ri->dma_base);
+ region_size = page_count * region_page_size;
+
+ err = pds_vfio_dirty_alloc_bitmaps(region,
+ page_count / BITS_PER_BYTE);
+ if (err) {
+ dev_err(&pdev->dev, "Failed to alloc dirty bitmaps: %pe\n",
+ ERR_PTR(err));
+ goto out_free_regions;
+ }
- err = __pds_vfio_dirty_alloc_sgl(pds_vfio, &dirty->host_ack,
- page_count);
- if (err) {
- __pds_vfio_dirty_free_sgl(pds_vfio, &dirty->host_seq);
- return err;
+ err = pds_vfio_dirty_alloc_sgl(pds_vfio, region, page_count);
+ if (err) {
+ dev_err(&pdev->dev, "Failed to alloc dirty sg lists: %pe\n",
+ ERR_PTR(err));
+ goto out_free_regions;
+ }
+
+ region->size = region_size;
+ region->start = region_start;
+ region->page_size = region_page_size;
+ region->dev_bmp_offset_start_byte = dev_bmp_offset_byte;
+
+ dev_bmp_offset_byte += page_count / BITS_PER_BYTE;
+ if (dev_bmp_offset_byte % BITS_PER_BYTE) {
+ dev_err(&pdev->dev, "Device bitmap offset is mis-aligned\n");
+ err = -EINVAL;
+ goto out_free_regions;
+ }
}
return 0;
+
+out_free_regions:
+ pds_vfio_dirty_free_bitmaps(dirty);
+ pds_vfio_dirty_free_sgl(pds_vfio);
+ pds_vfio_dirty_free_regions(dirty);
+
+ return err;
}
static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio,
@@ -181,16 +246,14 @@ static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio,
{
struct pci_dev *pdev = pds_vfio->vfio_coredev.pdev;
struct device *pdsc_dev = &pci_physfn(pdev)->dev;
- struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
- u64 region_start, region_size, region_page_size;
struct pds_lm_dirty_region_info *region_info;
struct interval_tree_node *node = NULL;
+ u64 region_page_size = *page_size;
u8 max_regions = 0, num_regions;
dma_addr_t regions_dma = 0;
u32 num_ranges = nnodes;
- u32 page_count;
- u16 len;
int err;
+ u16 len;
dev_dbg(&pdev->dev, "vf%u: Start dirty page tracking\n",
pds_vfio->vf_id);
@@ -217,39 +280,38 @@ static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio,
return -EOPNOTSUPP;
}
- /*
- * Only support 1 region for now. If there are any large gaps in the
- * VM's address regions, then this would be a waste of memory as we are
- * generating 2 bitmaps (ack/seq) from the min address to the max
- * address of the VM's address regions. In the future, if we support
- * more than one region in the device/driver we can split the bitmaps
- * on the largest address region gaps. We can do this split up to the
- * max_regions times returned from the dirty_status command.
- */
- max_regions = 1;
if (num_ranges > max_regions) {
vfio_combine_iova_ranges(ranges, nnodes, max_regions);
num_ranges = max_regions;
}
+ region_info = kcalloc(num_ranges, sizeof(*region_info), GFP_KERNEL);
+ if (!region_info)
+ return -ENOMEM;
+ len = num_ranges * sizeof(*region_info);
+
node = interval_tree_iter_first(ranges, 0, ULONG_MAX);
if (!node)
return -EINVAL;
+ for (int i = 0; i < num_ranges; i++) {
+ struct pds_lm_dirty_region_info *ri = &region_info[i];
+ u64 region_size = node->last - node->start + 1;
+ u64 region_start = node->start;
+ u32 page_count;
- region_size = node->last - node->start + 1;
- region_start = node->start;
- region_page_size = *page_size;
+ page_count = DIV_ROUND_UP(region_size, region_page_size);
- len = sizeof(*region_info);
- region_info = kzalloc(len, GFP_KERNEL);
- if (!region_info)
- return -ENOMEM;
+ ri->dma_base = cpu_to_le64(region_start);
+ ri->page_count = cpu_to_le32(page_count);
+ ri->page_size_log2 = ilog2(region_page_size);
- page_count = DIV_ROUND_UP(region_size, region_page_size);
+ dev_dbg(&pdev->dev,
+ "region_info[%d]: region_start 0x%llx region_end 0x%lx region_size 0x%llx page_count %u page_size %llu\n",
+ i, region_start, node->last, region_size, page_count,
+ region_page_size);
- region_info->dma_base = cpu_to_le64(region_start);
- region_info->page_count = cpu_to_le32(page_count);
- region_info->page_size_log2 = ilog2(region_page_size);
+ node = interval_tree_iter_next(node, 0, ULONG_MAX);
+ }
regions_dma = dma_map_single(pdsc_dev, (void *)region_info, len,
DMA_BIDIRECTIONAL);
@@ -258,39 +320,20 @@ static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio,
goto out_free_region_info;
}
- err = pds_vfio_dirty_enable_cmd(pds_vfio, regions_dma, max_regions);
+ err = pds_vfio_dirty_enable_cmd(pds_vfio, regions_dma, num_ranges);
dma_unmap_single(pdsc_dev, regions_dma, len, DMA_BIDIRECTIONAL);
if (err)
goto out_free_region_info;
- /*
- * page_count might be adjusted by the device,
- * update it before freeing region_info DMA
- */
- page_count = le32_to_cpu(region_info->page_count);
-
- dev_dbg(&pdev->dev,
- "region_info: regions_dma 0x%llx dma_base 0x%llx page_count %u page_size_log2 %u\n",
- regions_dma, region_start, page_count,
- (u8)ilog2(region_page_size));
-
- err = pds_vfio_dirty_alloc_bitmaps(dirty, page_count / BITS_PER_BYTE);
- if (err) {
- dev_err(&pdev->dev, "Failed to alloc dirty bitmaps: %pe\n",
- ERR_PTR(err));
- goto out_free_region_info;
- }
-
- err = pds_vfio_dirty_alloc_sgl(pds_vfio, page_count);
+ err = pds_vfio_dirty_alloc_regions(pds_vfio, region_info,
+ region_page_size, num_ranges);
if (err) {
- dev_err(&pdev->dev, "Failed to alloc dirty sg lists: %pe\n",
- ERR_PTR(err));
- goto out_free_bitmaps;
+ dev_err(&pdev->dev,
+ "Failed to allocate %d regions for tracking dirty regions: %pe\n",
+ num_regions, ERR_PTR(err));
+ goto out_dirty_disable;
}
- dirty->region_start = region_start;
- dirty->region_size = region_size;
- dirty->region_page_size = region_page_size;
pds_vfio_dirty_set_enabled(pds_vfio);
pds_vfio_print_guest_region_info(pds_vfio, max_regions);
@@ -299,8 +342,8 @@ static int pds_vfio_dirty_enable(struct pds_vfio_pci_device *pds_vfio,
return 0;
-out_free_bitmaps:
- pds_vfio_dirty_free_bitmaps(dirty);
+out_dirty_disable:
+ pds_vfio_dirty_disable_cmd(pds_vfio);
out_free_region_info:
kfree(region_info);
return err;
@@ -314,6 +357,7 @@ void pds_vfio_dirty_disable(struct pds_vfio_pci_device *pds_vfio, bool send_cmd)
pds_vfio_dirty_disable_cmd(pds_vfio);
pds_vfio_dirty_free_sgl(pds_vfio);
pds_vfio_dirty_free_bitmaps(&pds_vfio->dirty);
+ pds_vfio_dirty_free_regions(&pds_vfio->dirty);
}
if (send_cmd)
@@ -321,8 +365,9 @@ void pds_vfio_dirty_disable(struct pds_vfio_pci_device *pds_vfio, bool send_cmd)
}
static int pds_vfio_dirty_seq_ack(struct pds_vfio_pci_device *pds_vfio,
- struct pds_vfio_bmp_info *bmp_info,
- u32 offset, u32 bmp_bytes, bool read_seq)
+ struct pds_vfio_region *region,
+ unsigned long *seq_ack_bmp, u32 offset,
+ u32 bmp_bytes, bool read_seq)
{
const char *bmp_type_str = read_seq ? "read_seq" : "write_ack";
u8 dma_dir = read_seq ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
@@ -339,7 +384,7 @@ static int pds_vfio_dirty_seq_ack(struct pds_vfio_pci_device *pds_vfio,
int err;
int i;
- bmp = (void *)((u64)bmp_info->bmp + offset);
+ bmp = (void *)((u64)seq_ack_bmp + offset);
page_offset = offset_in_page(bmp);
bmp -= page_offset;
@@ -375,7 +420,7 @@ static int pds_vfio_dirty_seq_ack(struct pds_vfio_pci_device *pds_vfio,
goto out_free_sg_table;
for_each_sgtable_dma_sg(&sg_table, sg, i) {
- struct pds_lm_sg_elem *sg_elem = &bmp_info->sgl[i];
+ struct pds_lm_sg_elem *sg_elem = &region->sgl[i];
sg_elem->addr = cpu_to_le64(sg_dma_address(sg));
sg_elem->len = cpu_to_le32(sg_dma_len(sg));
@@ -383,15 +428,16 @@ static int pds_vfio_dirty_seq_ack(struct pds_vfio_pci_device *pds_vfio,
num_sge = sg_table.nents;
size = num_sge * sizeof(struct pds_lm_sg_elem);
- dma_sync_single_for_device(pdsc_dev, bmp_info->sgl_addr, size, dma_dir);
- err = pds_vfio_dirty_seq_ack_cmd(pds_vfio, bmp_info->sgl_addr, num_sge,
+ offset += region->dev_bmp_offset_start_byte;
+ dma_sync_single_for_device(pdsc_dev, region->sgl_addr, size, dma_dir);
+ err = pds_vfio_dirty_seq_ack_cmd(pds_vfio, region->sgl_addr, num_sge,
offset, bmp_bytes, read_seq);
if (err)
dev_err(&pdev->dev,
"Dirty bitmap %s failed offset %u bmp_bytes %u num_sge %u DMA 0x%llx: %pe\n",
bmp_type_str, offset, bmp_bytes,
- num_sge, bmp_info->sgl_addr, ERR_PTR(err));
- dma_sync_single_for_cpu(pdsc_dev, bmp_info->sgl_addr, size, dma_dir);
+ num_sge, region->sgl_addr, ERR_PTR(err));
+ dma_sync_single_for_cpu(pdsc_dev, region->sgl_addr, size, dma_dir);
dma_unmap_sgtable(pdsc_dev, &sg_table, dma_dir, 0);
out_free_sg_table:
@@ -403,32 +449,36 @@ out_free_pages:
}
static int pds_vfio_dirty_write_ack(struct pds_vfio_pci_device *pds_vfio,
+ struct pds_vfio_region *region,
u32 offset, u32 len)
{
- return pds_vfio_dirty_seq_ack(pds_vfio, &pds_vfio->dirty.host_ack,
+
+ return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_ack,
offset, len, WRITE_ACK);
}
static int pds_vfio_dirty_read_seq(struct pds_vfio_pci_device *pds_vfio,
+ struct pds_vfio_region *region,
u32 offset, u32 len)
{
- return pds_vfio_dirty_seq_ack(pds_vfio, &pds_vfio->dirty.host_seq,
+ return pds_vfio_dirty_seq_ack(pds_vfio, region, region->host_seq,
offset, len, READ_SEQ);
}
static int pds_vfio_dirty_process_bitmaps(struct pds_vfio_pci_device *pds_vfio,
+ struct pds_vfio_region *region,
struct iova_bitmap *dirty_bitmap,
u32 bmp_offset, u32 len_bytes)
{
- u64 page_size = pds_vfio->dirty.region_page_size;
- u64 region_start = pds_vfio->dirty.region_start;
+ u64 page_size = region->page_size;
+ u64 region_start = region->start;
u32 bmp_offset_bit;
__le64 *seq, *ack;
int dword_count;
dword_count = len_bytes / sizeof(u64);
- seq = (__le64 *)((u64)pds_vfio->dirty.host_seq.bmp + bmp_offset);
- ack = (__le64 *)((u64)pds_vfio->dirty.host_ack.bmp + bmp_offset);
+ seq = (__le64 *)((u64)region->host_seq + bmp_offset);
+ ack = (__le64 *)((u64)region->host_ack + bmp_offset);
bmp_offset_bit = bmp_offset * 8;
for (int i = 0; i < dword_count; i++) {
@@ -451,12 +501,28 @@ static int pds_vfio_dirty_process_bitmaps(struct pds_vfio_pci_device *pds_vfio,
return 0;
}
+static struct pds_vfio_region *
+pds_vfio_get_region(struct pds_vfio_pci_device *pds_vfio, unsigned long iova)
+{
+ struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
+
+ for (int i = 0; i < dirty->num_regions; i++) {
+ struct pds_vfio_region *region = &dirty->regions[i];
+
+ if (iova >= region->start &&
+ iova < (region->start + region->size))
+ return region;
+ }
+
+ return NULL;
+}
+
static int pds_vfio_dirty_sync(struct pds_vfio_pci_device *pds_vfio,
struct iova_bitmap *dirty_bitmap,
unsigned long iova, unsigned long length)
{
struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
- struct pds_vfio_dirty *dirty = &pds_vfio->dirty;
+ struct pds_vfio_region *region;
u64 bmp_offset, bmp_bytes;
u64 bitmap_size, pages;
int err;
@@ -469,25 +535,31 @@ static int pds_vfio_dirty_sync(struct pds_vfio_pci_device *pds_vfio,
return -EINVAL;
}
- pages = DIV_ROUND_UP(length, pds_vfio->dirty.region_page_size);
+ region = pds_vfio_get_region(pds_vfio, iova);
+ if (!region) {
+ dev_err(dev, "vf%u: Failed to find region that contains iova 0x%lx length 0x%lx\n",
+ pds_vfio->vf_id, iova, length);
+ return -EINVAL;
+ }
+
+ pages = DIV_ROUND_UP(length, region->page_size);
bitmap_size =
round_up(pages, sizeof(u64) * BITS_PER_BYTE) / BITS_PER_BYTE;
dev_dbg(dev,
"vf%u: iova 0x%lx length %lu page_size %llu pages %llu bitmap_size %llu\n",
- pds_vfio->vf_id, iova, length, pds_vfio->dirty.region_page_size,
+ pds_vfio->vf_id, iova, length, region->page_size,
pages, bitmap_size);
- if (!length || ((iova - dirty->region_start + length) > dirty->region_size)) {
+ if (!length || ((iova - region->start + length) > region->size)) {
dev_err(dev, "Invalid iova 0x%lx and/or length 0x%lx to sync\n",
iova, length);
return -EINVAL;
}
/* bitmap is modified in 64 bit chunks */
- bmp_bytes = ALIGN(DIV_ROUND_UP(length / dirty->region_page_size,
- sizeof(u64)),
- sizeof(u64));
+ bmp_bytes = ALIGN(DIV_ROUND_UP(length / region->page_size,
+ sizeof(u64)), sizeof(u64));
if (bmp_bytes != bitmap_size) {
dev_err(dev,
"Calculated bitmap bytes %llu not equal to bitmap size %llu\n",
@@ -495,23 +567,30 @@ static int pds_vfio_dirty_sync(struct pds_vfio_pci_device *pds_vfio,
return -EINVAL;
}
- bmp_offset = DIV_ROUND_UP((iova - dirty->region_start) /
- dirty->region_page_size, sizeof(u64));
+ if (bmp_bytes > region->bmp_bytes) {
+ dev_err(dev,
+ "Calculated bitmap bytes %llu larger than region's cached bmp_bytes %llu\n",
+ bmp_bytes, region->bmp_bytes);
+ return -EINVAL;
+ }
+
+ bmp_offset = DIV_ROUND_UP((iova - region->start) /
+ region->page_size, sizeof(u64));
dev_dbg(dev,
"Syncing dirty bitmap, iova 0x%lx length 0x%lx, bmp_offset %llu bmp_bytes %llu\n",
iova, length, bmp_offset, bmp_bytes);
- err = pds_vfio_dirty_read_seq(pds_vfio, bmp_offset, bmp_bytes);
+ err = pds_vfio_dirty_read_seq(pds_vfio, region, bmp_offset, bmp_bytes);
if (err)
return err;
- err = pds_vfio_dirty_process_bitmaps(pds_vfio, dirty_bitmap, bmp_offset,
- bmp_bytes);
+ err = pds_vfio_dirty_process_bitmaps(pds_vfio, region, dirty_bitmap,
+ bmp_offset, bmp_bytes);
if (err)
return err;
- err = pds_vfio_dirty_write_ack(pds_vfio, bmp_offset, bmp_bytes);
+ err = pds_vfio_dirty_write_ack(pds_vfio, region, bmp_offset, bmp_bytes);
if (err)
return err;
diff --git a/drivers/vfio/pci/pds/dirty.h b/drivers/vfio/pci/pds/dirty.h
index f78da25d75..c8e23018b8 100644
--- a/drivers/vfio/pci/pds/dirty.h
+++ b/drivers/vfio/pci/pds/dirty.h
@@ -4,20 +4,22 @@
#ifndef _DIRTY_H_
#define _DIRTY_H_
-struct pds_vfio_bmp_info {
- unsigned long *bmp;
- u32 bmp_bytes;
+struct pds_vfio_region {
+ unsigned long *host_seq;
+ unsigned long *host_ack;
+ u64 bmp_bytes;
+ u64 size;
+ u64 start;
+ u64 page_size;
struct pds_lm_sg_elem *sgl;
dma_addr_t sgl_addr;
+ u32 dev_bmp_offset_start_byte;
u16 num_sge;
};
struct pds_vfio_dirty {
- struct pds_vfio_bmp_info host_seq;
- struct pds_vfio_bmp_info host_ack;
- u64 region_size;
- u64 region_start;
- u64 region_page_size;
+ struct pds_vfio_region *regions;
+ u8 num_regions;
bool is_enabled;
};
diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c
index e27de61ac9..07fea08ea8 100644
--- a/drivers/vfio/pci/vfio_pci_rdwr.c
+++ b/drivers/vfio/pci/vfio_pci_rdwr.c
@@ -38,7 +38,7 @@
#define vfio_iowrite8 iowrite8
#define VFIO_IOWRITE(size) \
-static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \
+int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev, \
bool test_mem, u##size val, void __iomem *io) \
{ \
if (test_mem) { \
@@ -55,7 +55,8 @@ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \
up_read(&vdev->memory_lock); \
\
return 0; \
-}
+} \
+EXPORT_SYMBOL_GPL(vfio_pci_core_iowrite##size);
VFIO_IOWRITE(8)
VFIO_IOWRITE(16)
@@ -65,7 +66,7 @@ VFIO_IOWRITE(64)
#endif
#define VFIO_IOREAD(size) \
-static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \
+int vfio_pci_core_ioread##size(struct vfio_pci_core_device *vdev, \
bool test_mem, u##size *val, void __iomem *io) \
{ \
if (test_mem) { \
@@ -82,7 +83,8 @@ static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \
up_read(&vdev->memory_lock); \
\
return 0; \
-}
+} \
+EXPORT_SYMBOL_GPL(vfio_pci_core_ioread##size);
VFIO_IOREAD(8)
VFIO_IOREAD(16)
@@ -119,13 +121,13 @@ static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
if (copy_from_user(&val, buf, 4))
return -EFAULT;
- ret = vfio_pci_iowrite32(vdev, test_mem,
- val, io + off);
+ ret = vfio_pci_core_iowrite32(vdev, test_mem,
+ val, io + off);
if (ret)
return ret;
} else {
- ret = vfio_pci_ioread32(vdev, test_mem,
- &val, io + off);
+ ret = vfio_pci_core_ioread32(vdev, test_mem,
+ &val, io + off);
if (ret)
return ret;
@@ -141,13 +143,13 @@ static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
if (copy_from_user(&val, buf, 2))
return -EFAULT;
- ret = vfio_pci_iowrite16(vdev, test_mem,
- val, io + off);
+ ret = vfio_pci_core_iowrite16(vdev, test_mem,
+ val, io + off);
if (ret)
return ret;
} else {
- ret = vfio_pci_ioread16(vdev, test_mem,
- &val, io + off);
+ ret = vfio_pci_core_ioread16(vdev, test_mem,
+ &val, io + off);
if (ret)
return ret;
@@ -163,13 +165,13 @@ static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
if (copy_from_user(&val, buf, 1))
return -EFAULT;
- ret = vfio_pci_iowrite8(vdev, test_mem,
- val, io + off);
+ ret = vfio_pci_core_iowrite8(vdev, test_mem,
+ val, io + off);
if (ret)
return ret;
} else {
- ret = vfio_pci_ioread8(vdev, test_mem,
- &val, io + off);
+ ret = vfio_pci_core_ioread8(vdev, test_mem,
+ &val, io + off);
if (ret)
return ret;
@@ -200,7 +202,7 @@ static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
return done;
}
-static int vfio_pci_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
+int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
{
struct pci_dev *pdev = vdev->pdev;
int ret;
@@ -223,6 +225,7 @@ static int vfio_pci_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
return 0;
}
+EXPORT_SYMBOL_GPL(vfio_pci_core_setup_barmap);
ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
size_t count, loff_t *ppos, bool iswrite)
@@ -262,7 +265,7 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
}
x_end = end;
} else {
- int ret = vfio_pci_setup_barmap(vdev, bar);
+ int ret = vfio_pci_core_setup_barmap(vdev, bar);
if (ret) {
done = ret;
goto out;
@@ -363,21 +366,21 @@ static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd,
{
switch (ioeventfd->count) {
case 1:
- vfio_pci_iowrite8(ioeventfd->vdev, test_mem,
- ioeventfd->data, ioeventfd->addr);
+ vfio_pci_core_iowrite8(ioeventfd->vdev, test_mem,
+ ioeventfd->data, ioeventfd->addr);
break;
case 2:
- vfio_pci_iowrite16(ioeventfd->vdev, test_mem,
- ioeventfd->data, ioeventfd->addr);
+ vfio_pci_core_iowrite16(ioeventfd->vdev, test_mem,
+ ioeventfd->data, ioeventfd->addr);
break;
case 4:
- vfio_pci_iowrite32(ioeventfd->vdev, test_mem,
- ioeventfd->data, ioeventfd->addr);
+ vfio_pci_core_iowrite32(ioeventfd->vdev, test_mem,
+ ioeventfd->data, ioeventfd->addr);
break;
#ifdef iowrite64
case 8:
- vfio_pci_iowrite64(ioeventfd->vdev, test_mem,
- ioeventfd->data, ioeventfd->addr);
+ vfio_pci_core_iowrite64(ioeventfd->vdev, test_mem,
+ ioeventfd->data, ioeventfd->addr);
break;
#endif
}
@@ -438,7 +441,7 @@ int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
return -EINVAL;
#endif
- ret = vfio_pci_setup_barmap(vdev, bar);
+ ret = vfio_pci_core_setup_barmap(vdev, bar);
if (ret)
return ret;
diff --git a/drivers/vfio/pci/virtio/Kconfig b/drivers/vfio/pci/virtio/Kconfig
new file mode 100644
index 0000000000..bd80eca4a1
--- /dev/null
+++ b/drivers/vfio/pci/virtio/Kconfig
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VIRTIO_VFIO_PCI
+ tristate "VFIO support for VIRTIO NET PCI devices"
+ depends on VIRTIO_PCI && VIRTIO_PCI_ADMIN_LEGACY
+ select VFIO_PCI_CORE
+ help
+ This provides support for exposing VIRTIO NET VF devices which support
+ legacy IO access, using the VFIO framework that can work with a legacy
+ virtio driver in the guest.
+ Based on PCIe spec, VFs do not support I/O Space.
+ As of that this driver emulates I/O BAR in software to let a VF be
+ seen as a transitional device by its users and let it work with
+ a legacy driver.
+
+ If you don't know what to do here, say N.
diff --git a/drivers/vfio/pci/virtio/Makefile b/drivers/vfio/pci/virtio/Makefile
new file mode 100644
index 0000000000..7171105baf
--- /dev/null
+++ b/drivers/vfio/pci/virtio/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_VIRTIO_VFIO_PCI) += virtio-vfio-pci.o
+virtio-vfio-pci-y := main.o
diff --git a/drivers/vfio/pci/virtio/main.c b/drivers/vfio/pci/virtio/main.c
new file mode 100644
index 0000000000..d5af683837
--- /dev/null
+++ b/drivers/vfio/pci/virtio/main.c
@@ -0,0 +1,576 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/vfio.h>
+#include <linux/vfio_pci_core.h>
+#include <linux/virtio_pci.h>
+#include <linux/virtio_net.h>
+#include <linux/virtio_pci_admin.h>
+
+struct virtiovf_pci_core_device {
+ struct vfio_pci_core_device core_device;
+ u8 *bar0_virtual_buf;
+ /* synchronize access to the virtual buf */
+ struct mutex bar_mutex;
+ void __iomem *notify_addr;
+ u64 notify_offset;
+ __le32 pci_base_addr_0;
+ __le16 pci_cmd;
+ u8 bar0_virtual_buf_size;
+ u8 notify_bar;
+};
+
+static int
+virtiovf_issue_legacy_rw_cmd(struct virtiovf_pci_core_device *virtvdev,
+ loff_t pos, char __user *buf,
+ size_t count, bool read)
+{
+ bool msix_enabled =
+ (virtvdev->core_device.irq_type == VFIO_PCI_MSIX_IRQ_INDEX);
+ struct pci_dev *pdev = virtvdev->core_device.pdev;
+ u8 *bar0_buf = virtvdev->bar0_virtual_buf;
+ bool common;
+ u8 offset;
+ int ret;
+
+ common = pos < VIRTIO_PCI_CONFIG_OFF(msix_enabled);
+ /* offset within the relevant configuration area */
+ offset = common ? pos : pos - VIRTIO_PCI_CONFIG_OFF(msix_enabled);
+ mutex_lock(&virtvdev->bar_mutex);
+ if (read) {
+ if (common)
+ ret = virtio_pci_admin_legacy_common_io_read(pdev, offset,
+ count, bar0_buf + pos);
+ else
+ ret = virtio_pci_admin_legacy_device_io_read(pdev, offset,
+ count, bar0_buf + pos);
+ if (ret)
+ goto out;
+ if (copy_to_user(buf, bar0_buf + pos, count))
+ ret = -EFAULT;
+ } else {
+ if (copy_from_user(bar0_buf + pos, buf, count)) {
+ ret = -EFAULT;
+ goto out;
+ }
+
+ if (common)
+ ret = virtio_pci_admin_legacy_common_io_write(pdev, offset,
+ count, bar0_buf + pos);
+ else
+ ret = virtio_pci_admin_legacy_device_io_write(pdev, offset,
+ count, bar0_buf + pos);
+ }
+out:
+ mutex_unlock(&virtvdev->bar_mutex);
+ return ret;
+}
+
+static int
+virtiovf_pci_bar0_rw(struct virtiovf_pci_core_device *virtvdev,
+ loff_t pos, char __user *buf,
+ size_t count, bool read)
+{
+ struct vfio_pci_core_device *core_device = &virtvdev->core_device;
+ struct pci_dev *pdev = core_device->pdev;
+ u16 queue_notify;
+ int ret;
+
+ if (!(le16_to_cpu(virtvdev->pci_cmd) & PCI_COMMAND_IO))
+ return -EIO;
+
+ if (pos + count > virtvdev->bar0_virtual_buf_size)
+ return -EINVAL;
+
+ ret = pm_runtime_resume_and_get(&pdev->dev);
+ if (ret) {
+ pci_info_ratelimited(pdev, "runtime resume failed %d\n", ret);
+ return -EIO;
+ }
+
+ switch (pos) {
+ case VIRTIO_PCI_QUEUE_NOTIFY:
+ if (count != sizeof(queue_notify)) {
+ ret = -EINVAL;
+ goto end;
+ }
+ if (read) {
+ ret = vfio_pci_core_ioread16(core_device, true, &queue_notify,
+ virtvdev->notify_addr);
+ if (ret)
+ goto end;
+ if (copy_to_user(buf, &queue_notify,
+ sizeof(queue_notify))) {
+ ret = -EFAULT;
+ goto end;
+ }
+ } else {
+ if (copy_from_user(&queue_notify, buf, count)) {
+ ret = -EFAULT;
+ goto end;
+ }
+ ret = vfio_pci_core_iowrite16(core_device, true, queue_notify,
+ virtvdev->notify_addr);
+ }
+ break;
+ default:
+ ret = virtiovf_issue_legacy_rw_cmd(virtvdev, pos, buf, count,
+ read);
+ }
+
+end:
+ pm_runtime_put(&pdev->dev);
+ return ret ? ret : count;
+}
+
+static bool range_intersect_range(loff_t range1_start, size_t count1,
+ loff_t range2_start, size_t count2,
+ loff_t *start_offset,
+ size_t *intersect_count,
+ size_t *register_offset)
+{
+ if (range1_start <= range2_start &&
+ range1_start + count1 > range2_start) {
+ *start_offset = range2_start - range1_start;
+ *intersect_count = min_t(size_t, count2,
+ range1_start + count1 - range2_start);
+ *register_offset = 0;
+ return true;
+ }
+
+ if (range1_start > range2_start &&
+ range1_start < range2_start + count2) {
+ *start_offset = 0;
+ *intersect_count = min_t(size_t, count1,
+ range2_start + count2 - range1_start);
+ *register_offset = range1_start - range2_start;
+ return true;
+ }
+
+ return false;
+}
+
+static ssize_t virtiovf_pci_read_config(struct vfio_device *core_vdev,
+ char __user *buf, size_t count,
+ loff_t *ppos)
+{
+ struct virtiovf_pci_core_device *virtvdev = container_of(
+ core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
+ loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
+ size_t register_offset;
+ loff_t copy_offset;
+ size_t copy_count;
+ __le32 val32;
+ __le16 val16;
+ u8 val8;
+ int ret;
+
+ ret = vfio_pci_core_read(core_vdev, buf, count, ppos);
+ if (ret < 0)
+ return ret;
+
+ if (range_intersect_range(pos, count, PCI_DEVICE_ID, sizeof(val16),
+ &copy_offset, &copy_count, &register_offset)) {
+ val16 = cpu_to_le16(VIRTIO_TRANS_ID_NET);
+ if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset, copy_count))
+ return -EFAULT;
+ }
+
+ if ((le16_to_cpu(virtvdev->pci_cmd) & PCI_COMMAND_IO) &&
+ range_intersect_range(pos, count, PCI_COMMAND, sizeof(val16),
+ &copy_offset, &copy_count, &register_offset)) {
+ if (copy_from_user((void *)&val16 + register_offset, buf + copy_offset,
+ copy_count))
+ return -EFAULT;
+ val16 |= cpu_to_le16(PCI_COMMAND_IO);
+ if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset,
+ copy_count))
+ return -EFAULT;
+ }
+
+ if (range_intersect_range(pos, count, PCI_REVISION_ID, sizeof(val8),
+ &copy_offset, &copy_count, &register_offset)) {
+ /* Transional needs to have revision 0 */
+ val8 = 0;
+ if (copy_to_user(buf + copy_offset, &val8, copy_count))
+ return -EFAULT;
+ }
+
+ if (range_intersect_range(pos, count, PCI_BASE_ADDRESS_0, sizeof(val32),
+ &copy_offset, &copy_count, &register_offset)) {
+ u32 bar_mask = ~(virtvdev->bar0_virtual_buf_size - 1);
+ u32 pci_base_addr_0 = le32_to_cpu(virtvdev->pci_base_addr_0);
+
+ val32 = cpu_to_le32((pci_base_addr_0 & bar_mask) | PCI_BASE_ADDRESS_SPACE_IO);
+ if (copy_to_user(buf + copy_offset, (void *)&val32 + register_offset, copy_count))
+ return -EFAULT;
+ }
+
+ if (range_intersect_range(pos, count, PCI_SUBSYSTEM_ID, sizeof(val16),
+ &copy_offset, &copy_count, &register_offset)) {
+ /*
+ * Transitional devices use the PCI subsystem device id as
+ * virtio device id, same as legacy driver always did.
+ */
+ val16 = cpu_to_le16(VIRTIO_ID_NET);
+ if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset,
+ copy_count))
+ return -EFAULT;
+ }
+
+ if (range_intersect_range(pos, count, PCI_SUBSYSTEM_VENDOR_ID, sizeof(val16),
+ &copy_offset, &copy_count, &register_offset)) {
+ val16 = cpu_to_le16(PCI_VENDOR_ID_REDHAT_QUMRANET);
+ if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset,
+ copy_count))
+ return -EFAULT;
+ }
+
+ return count;
+}
+
+static ssize_t
+virtiovf_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct virtiovf_pci_core_device *virtvdev = container_of(
+ core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
+ unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
+ loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
+
+ if (!count)
+ return 0;
+
+ if (index == VFIO_PCI_CONFIG_REGION_INDEX)
+ return virtiovf_pci_read_config(core_vdev, buf, count, ppos);
+
+ if (index == VFIO_PCI_BAR0_REGION_INDEX)
+ return virtiovf_pci_bar0_rw(virtvdev, pos, buf, count, true);
+
+ return vfio_pci_core_read(core_vdev, buf, count, ppos);
+}
+
+static ssize_t virtiovf_pci_write_config(struct vfio_device *core_vdev,
+ const char __user *buf, size_t count,
+ loff_t *ppos)
+{
+ struct virtiovf_pci_core_device *virtvdev = container_of(
+ core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
+ loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
+ size_t register_offset;
+ loff_t copy_offset;
+ size_t copy_count;
+
+ if (range_intersect_range(pos, count, PCI_COMMAND, sizeof(virtvdev->pci_cmd),
+ &copy_offset, &copy_count,
+ &register_offset)) {
+ if (copy_from_user((void *)&virtvdev->pci_cmd + register_offset,
+ buf + copy_offset,
+ copy_count))
+ return -EFAULT;
+ }
+
+ if (range_intersect_range(pos, count, PCI_BASE_ADDRESS_0,
+ sizeof(virtvdev->pci_base_addr_0),
+ &copy_offset, &copy_count,
+ &register_offset)) {
+ if (copy_from_user((void *)&virtvdev->pci_base_addr_0 + register_offset,
+ buf + copy_offset,
+ copy_count))
+ return -EFAULT;
+ }
+
+ return vfio_pci_core_write(core_vdev, buf, count, ppos);
+}
+
+static ssize_t
+virtiovf_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct virtiovf_pci_core_device *virtvdev = container_of(
+ core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
+ unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
+ loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
+
+ if (!count)
+ return 0;
+
+ if (index == VFIO_PCI_CONFIG_REGION_INDEX)
+ return virtiovf_pci_write_config(core_vdev, buf, count, ppos);
+
+ if (index == VFIO_PCI_BAR0_REGION_INDEX)
+ return virtiovf_pci_bar0_rw(virtvdev, pos, (char __user *)buf, count, false);
+
+ return vfio_pci_core_write(core_vdev, buf, count, ppos);
+}
+
+static int
+virtiovf_pci_ioctl_get_region_info(struct vfio_device *core_vdev,
+ unsigned int cmd, unsigned long arg)
+{
+ struct virtiovf_pci_core_device *virtvdev = container_of(
+ core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
+ unsigned long minsz = offsetofend(struct vfio_region_info, offset);
+ void __user *uarg = (void __user *)arg;
+ struct vfio_region_info info = {};
+
+ if (copy_from_user(&info, uarg, minsz))
+ return -EFAULT;
+
+ if (info.argsz < minsz)
+ return -EINVAL;
+
+ switch (info.index) {
+ case VFIO_PCI_BAR0_REGION_INDEX:
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = virtvdev->bar0_virtual_buf_size;
+ info.flags = VFIO_REGION_INFO_FLAG_READ |
+ VFIO_REGION_INFO_FLAG_WRITE;
+ return copy_to_user(uarg, &info, minsz) ? -EFAULT : 0;
+ default:
+ return vfio_pci_core_ioctl(core_vdev, cmd, arg);
+ }
+}
+
+static long
+virtiovf_vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
+ unsigned long arg)
+{
+ switch (cmd) {
+ case VFIO_DEVICE_GET_REGION_INFO:
+ return virtiovf_pci_ioctl_get_region_info(core_vdev, cmd, arg);
+ default:
+ return vfio_pci_core_ioctl(core_vdev, cmd, arg);
+ }
+}
+
+static int
+virtiovf_set_notify_addr(struct virtiovf_pci_core_device *virtvdev)
+{
+ struct vfio_pci_core_device *core_device = &virtvdev->core_device;
+ int ret;
+
+ /*
+ * Setup the BAR where the 'notify' exists to be used by vfio as well
+ * This will let us mmap it only once and use it when needed.
+ */
+ ret = vfio_pci_core_setup_barmap(core_device,
+ virtvdev->notify_bar);
+ if (ret)
+ return ret;
+
+ virtvdev->notify_addr = core_device->barmap[virtvdev->notify_bar] +
+ virtvdev->notify_offset;
+ return 0;
+}
+
+static int virtiovf_pci_open_device(struct vfio_device *core_vdev)
+{
+ struct virtiovf_pci_core_device *virtvdev = container_of(
+ core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
+ struct vfio_pci_core_device *vdev = &virtvdev->core_device;
+ int ret;
+
+ ret = vfio_pci_core_enable(vdev);
+ if (ret)
+ return ret;
+
+ if (virtvdev->bar0_virtual_buf) {
+ /*
+ * Upon close_device() the vfio_pci_core_disable() is called
+ * and will close all the previous mmaps, so it seems that the
+ * valid life cycle for the 'notify' addr is per open/close.
+ */
+ ret = virtiovf_set_notify_addr(virtvdev);
+ if (ret) {
+ vfio_pci_core_disable(vdev);
+ return ret;
+ }
+ }
+
+ vfio_pci_core_finish_enable(vdev);
+ return 0;
+}
+
+static int virtiovf_get_device_config_size(unsigned short device)
+{
+ /* Network card */
+ return offsetofend(struct virtio_net_config, status);
+}
+
+static int virtiovf_read_notify_info(struct virtiovf_pci_core_device *virtvdev)
+{
+ u64 offset;
+ int ret;
+ u8 bar;
+
+ ret = virtio_pci_admin_legacy_io_notify_info(virtvdev->core_device.pdev,
+ VIRTIO_ADMIN_CMD_NOTIFY_INFO_FLAGS_OWNER_MEM,
+ &bar, &offset);
+ if (ret)
+ return ret;
+
+ virtvdev->notify_bar = bar;
+ virtvdev->notify_offset = offset;
+ return 0;
+}
+
+static int virtiovf_pci_init_device(struct vfio_device *core_vdev)
+{
+ struct virtiovf_pci_core_device *virtvdev = container_of(
+ core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
+ struct pci_dev *pdev;
+ int ret;
+
+ ret = vfio_pci_core_init_dev(core_vdev);
+ if (ret)
+ return ret;
+
+ pdev = virtvdev->core_device.pdev;
+ ret = virtiovf_read_notify_info(virtvdev);
+ if (ret)
+ return ret;
+
+ virtvdev->bar0_virtual_buf_size = VIRTIO_PCI_CONFIG_OFF(true) +
+ virtiovf_get_device_config_size(pdev->device);
+ BUILD_BUG_ON(!is_power_of_2(virtvdev->bar0_virtual_buf_size));
+ virtvdev->bar0_virtual_buf = kzalloc(virtvdev->bar0_virtual_buf_size,
+ GFP_KERNEL);
+ if (!virtvdev->bar0_virtual_buf)
+ return -ENOMEM;
+ mutex_init(&virtvdev->bar_mutex);
+ return 0;
+}
+
+static void virtiovf_pci_core_release_dev(struct vfio_device *core_vdev)
+{
+ struct virtiovf_pci_core_device *virtvdev = container_of(
+ core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
+
+ kfree(virtvdev->bar0_virtual_buf);
+ vfio_pci_core_release_dev(core_vdev);
+}
+
+static const struct vfio_device_ops virtiovf_vfio_pci_tran_ops = {
+ .name = "virtio-vfio-pci-trans",
+ .init = virtiovf_pci_init_device,
+ .release = virtiovf_pci_core_release_dev,
+ .open_device = virtiovf_pci_open_device,
+ .close_device = vfio_pci_core_close_device,
+ .ioctl = virtiovf_vfio_pci_core_ioctl,
+ .device_feature = vfio_pci_core_ioctl_feature,
+ .read = virtiovf_pci_core_read,
+ .write = virtiovf_pci_core_write,
+ .mmap = vfio_pci_core_mmap,
+ .request = vfio_pci_core_request,
+ .match = vfio_pci_core_match,
+ .bind_iommufd = vfio_iommufd_physical_bind,
+ .unbind_iommufd = vfio_iommufd_physical_unbind,
+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
+ .detach_ioas = vfio_iommufd_physical_detach_ioas,
+};
+
+static const struct vfio_device_ops virtiovf_vfio_pci_ops = {
+ .name = "virtio-vfio-pci",
+ .init = vfio_pci_core_init_dev,
+ .release = vfio_pci_core_release_dev,
+ .open_device = virtiovf_pci_open_device,
+ .close_device = vfio_pci_core_close_device,
+ .ioctl = vfio_pci_core_ioctl,
+ .device_feature = vfio_pci_core_ioctl_feature,
+ .read = vfio_pci_core_read,
+ .write = vfio_pci_core_write,
+ .mmap = vfio_pci_core_mmap,
+ .request = vfio_pci_core_request,
+ .match = vfio_pci_core_match,
+ .bind_iommufd = vfio_iommufd_physical_bind,
+ .unbind_iommufd = vfio_iommufd_physical_unbind,
+ .attach_ioas = vfio_iommufd_physical_attach_ioas,
+ .detach_ioas = vfio_iommufd_physical_detach_ioas,
+};
+
+static bool virtiovf_bar0_exists(struct pci_dev *pdev)
+{
+ struct resource *res = pdev->resource;
+
+ return res->flags;
+}
+
+static int virtiovf_pci_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+{
+ const struct vfio_device_ops *ops = &virtiovf_vfio_pci_ops;
+ struct virtiovf_pci_core_device *virtvdev;
+ int ret;
+
+ if (pdev->is_virtfn && virtio_pci_admin_has_legacy_io(pdev) &&
+ !virtiovf_bar0_exists(pdev))
+ ops = &virtiovf_vfio_pci_tran_ops;
+
+ virtvdev = vfio_alloc_device(virtiovf_pci_core_device, core_device.vdev,
+ &pdev->dev, ops);
+ if (IS_ERR(virtvdev))
+ return PTR_ERR(virtvdev);
+
+ dev_set_drvdata(&pdev->dev, &virtvdev->core_device);
+ ret = vfio_pci_core_register_device(&virtvdev->core_device);
+ if (ret)
+ goto out;
+ return 0;
+out:
+ vfio_put_device(&virtvdev->core_device.vdev);
+ return ret;
+}
+
+static void virtiovf_pci_remove(struct pci_dev *pdev)
+{
+ struct virtiovf_pci_core_device *virtvdev = dev_get_drvdata(&pdev->dev);
+
+ vfio_pci_core_unregister_device(&virtvdev->core_device);
+ vfio_put_device(&virtvdev->core_device.vdev);
+}
+
+static const struct pci_device_id virtiovf_pci_table[] = {
+ /* Only virtio-net is supported/tested so far */
+ { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_REDHAT_QUMRANET, 0x1041) },
+ {}
+};
+
+MODULE_DEVICE_TABLE(pci, virtiovf_pci_table);
+
+static void virtiovf_pci_aer_reset_done(struct pci_dev *pdev)
+{
+ struct virtiovf_pci_core_device *virtvdev = dev_get_drvdata(&pdev->dev);
+
+ virtvdev->pci_cmd = 0;
+}
+
+static const struct pci_error_handlers virtiovf_err_handlers = {
+ .reset_done = virtiovf_pci_aer_reset_done,
+ .error_detected = vfio_pci_core_aer_err_detected,
+};
+
+static struct pci_driver virtiovf_pci_driver = {
+ .name = KBUILD_MODNAME,
+ .id_table = virtiovf_pci_table,
+ .probe = virtiovf_pci_probe,
+ .remove = virtiovf_pci_remove,
+ .err_handler = &virtiovf_err_handlers,
+ .driver_managed_dma = true,
+};
+
+module_pci_driver(virtiovf_pci_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Yishai Hadas <yishaih@nvidia.com>");
+MODULE_DESCRIPTION(
+ "VIRTIO VFIO PCI - User Level meta-driver for VIRTIO NET devices");
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-06 02:17:12 +0000