summaryrefslogtreecommitdiffstats
path: root/drivers/vdpa
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/vdpa')
-rw-r--r--drivers/vdpa/mlx5/core/mlx5_vdpa.h32
-rw-r--r--drivers/vdpa/mlx5/core/mr.c209
-rw-r--r--drivers/vdpa/mlx5/core/resources.c6
-rw-r--r--drivers/vdpa/mlx5/net/mlx5_vnet.c137
-rw-r--r--drivers/vdpa/vdpa_sim/vdpa_sim.c52
-rw-r--r--drivers/vdpa/vdpa_user/vduse_dev.c40
6 files changed, 319 insertions, 157 deletions
diff --git a/drivers/vdpa/mlx5/core/mlx5_vdpa.h b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
index ca56242972..84547d998b 100644
--- a/drivers/vdpa/mlx5/core/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
@@ -31,11 +31,9 @@ struct mlx5_vdpa_mr {
struct list_head head;
unsigned long num_directs;
unsigned long num_klms;
- /* state of dvq mr */
- bool initialized;
- /* serialize mkey creation and destruction */
- struct mutex mkey_mtx;
+ struct vhost_iotlb *iotlb;
+
bool user_mr;
};
@@ -74,11 +72,12 @@ struct mlx5_vdpa_wq_ent {
enum {
MLX5_VDPA_DATAVQ_GROUP,
MLX5_VDPA_CVQ_GROUP,
+ MLX5_VDPA_DATAVQ_DESC_GROUP,
MLX5_VDPA_NUMVQ_GROUPS
};
enum {
- MLX5_VDPA_NUM_AS = MLX5_VDPA_NUMVQ_GROUPS
+ MLX5_VDPA_NUM_AS = 2
};
struct mlx5_vdpa_dev {
@@ -93,7 +92,9 @@ struct mlx5_vdpa_dev {
u16 max_idx;
u32 generation;
- struct mlx5_vdpa_mr mr;
+ struct mlx5_vdpa_mr *mr[MLX5_VDPA_NUM_AS];
+ /* serialize mr access */
+ struct mutex mr_mtx;
struct mlx5_control_vq cvq;
struct workqueue_struct *wq;
unsigned int group2asid[MLX5_VDPA_NUMVQ_GROUPS];
@@ -114,12 +115,19 @@ void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev);
int mlx5_vdpa_create_mkey(struct mlx5_vdpa_dev *mvdev, u32 *mkey, u32 *in,
int inlen);
int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, u32 mkey);
-int mlx5_vdpa_handle_set_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,
- bool *change_map, unsigned int asid);
-int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,
- unsigned int asid);
-void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev);
-void mlx5_vdpa_destroy_mr_asid(struct mlx5_vdpa_dev *mvdev, unsigned int asid);
+struct mlx5_vdpa_mr *mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
+ struct vhost_iotlb *iotlb);
+void mlx5_vdpa_destroy_mr_resources(struct mlx5_vdpa_dev *mvdev);
+void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev,
+ struct mlx5_vdpa_mr *mr);
+void mlx5_vdpa_update_mr(struct mlx5_vdpa_dev *mvdev,
+ struct mlx5_vdpa_mr *mr,
+ unsigned int asid);
+int mlx5_vdpa_update_cvq_iotlb(struct mlx5_vdpa_dev *mvdev,
+ struct vhost_iotlb *iotlb,
+ unsigned int asid);
+int mlx5_vdpa_create_dma_mr(struct mlx5_vdpa_dev *mvdev);
+int mlx5_vdpa_reset_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid);
#define mlx5_vdpa_warn(__dev, format, ...) \
dev_warn((__dev)->mdev->device, "%s:%d:(pid %d) warning: " format, __func__, __LINE__, \
diff --git a/drivers/vdpa/mlx5/core/mr.c b/drivers/vdpa/mlx5/core/mr.c
index 5a1971fcd8..2197c46e56 100644
--- a/drivers/vdpa/mlx5/core/mr.c
+++ b/drivers/vdpa/mlx5/core/mr.c
@@ -301,10 +301,13 @@ static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct
sg_free_table(&mr->sg_head);
}
-static int add_direct_chain(struct mlx5_vdpa_dev *mvdev, u64 start, u64 size, u8 perm,
+static int add_direct_chain(struct mlx5_vdpa_dev *mvdev,
+ struct mlx5_vdpa_mr *mr,
+ u64 start,
+ u64 size,
+ u8 perm,
struct vhost_iotlb *iotlb)
{
- struct mlx5_vdpa_mr *mr = &mvdev->mr;
struct mlx5_vdpa_direct_mr *dmr;
struct mlx5_vdpa_direct_mr *n;
LIST_HEAD(tmp);
@@ -354,9 +357,10 @@ err_alloc:
* indirect memory key that provides access to the enitre address space given
* by iotlb.
*/
-static int create_user_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)
+static int create_user_mr(struct mlx5_vdpa_dev *mvdev,
+ struct mlx5_vdpa_mr *mr,
+ struct vhost_iotlb *iotlb)
{
- struct mlx5_vdpa_mr *mr = &mvdev->mr;
struct mlx5_vdpa_direct_mr *dmr;
struct mlx5_vdpa_direct_mr *n;
struct vhost_iotlb_map *map;
@@ -384,7 +388,7 @@ static int create_user_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb
LOG_MAX_KLM_SIZE);
mr->num_klms += nnuls;
}
- err = add_direct_chain(mvdev, ps, pe - ps, pperm, iotlb);
+ err = add_direct_chain(mvdev, mr, ps, pe - ps, pperm, iotlb);
if (err)
goto err_chain;
}
@@ -393,7 +397,7 @@ static int create_user_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb
pperm = map->perm;
}
}
- err = add_direct_chain(mvdev, ps, pe - ps, pperm, iotlb);
+ err = add_direct_chain(mvdev, mr, ps, pe - ps, pperm, iotlb);
if (err)
goto err_chain;
@@ -450,20 +454,23 @@ static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
mlx5_vdpa_destroy_mkey(mvdev, mr->mkey);
}
-static int dup_iotlb(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *src)
+static int dup_iotlb(struct vhost_iotlb *dst, struct vhost_iotlb *src)
{
struct vhost_iotlb_map *map;
u64 start = 0, last = ULLONG_MAX;
int err;
+ if (dst == src)
+ return -EINVAL;
+
if (!src) {
- err = vhost_iotlb_add_range(mvdev->cvq.iotlb, start, last, start, VHOST_ACCESS_RW);
+ err = vhost_iotlb_add_range(dst, start, last, start, VHOST_ACCESS_RW);
return err;
}
for (map = vhost_iotlb_itree_first(src, start, last); map;
map = vhost_iotlb_itree_next(map, start, last)) {
- err = vhost_iotlb_add_range(mvdev->cvq.iotlb, map->start, map->last,
+ err = vhost_iotlb_add_range(dst, map->start, map->last,
map->addr, map->perm);
if (err)
return err;
@@ -471,9 +478,9 @@ static int dup_iotlb(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *src)
return 0;
}
-static void prune_iotlb(struct mlx5_vdpa_dev *mvdev)
+static void prune_iotlb(struct vhost_iotlb *iotlb)
{
- vhost_iotlb_del_range(mvdev->cvq.iotlb, 0, ULLONG_MAX);
+ vhost_iotlb_del_range(iotlb, 0, ULLONG_MAX);
}
static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
@@ -489,133 +496,169 @@ static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr
}
}
-static void _mlx5_vdpa_destroy_cvq_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
+static void _mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
{
- if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
- return;
+ if (mr->user_mr)
+ destroy_user_mr(mvdev, mr);
+ else
+ destroy_dma_mr(mvdev, mr);
- prune_iotlb(mvdev);
+ vhost_iotlb_free(mr->iotlb);
}
-static void _mlx5_vdpa_destroy_dvq_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
+void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev,
+ struct mlx5_vdpa_mr *mr)
{
- struct mlx5_vdpa_mr *mr = &mvdev->mr;
-
- if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] != asid)
+ if (!mr)
return;
- if (!mr->initialized)
- return;
+ mutex_lock(&mvdev->mr_mtx);
- if (mr->user_mr)
- destroy_user_mr(mvdev, mr);
- else
- destroy_dma_mr(mvdev, mr);
+ _mlx5_vdpa_destroy_mr(mvdev, mr);
+
+ for (int i = 0; i < MLX5_VDPA_NUM_AS; i++) {
+ if (mvdev->mr[i] == mr)
+ mvdev->mr[i] = NULL;
+ }
- mr->initialized = false;
+ mutex_unlock(&mvdev->mr_mtx);
+
+ kfree(mr);
}
-void mlx5_vdpa_destroy_mr_asid(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
+void mlx5_vdpa_update_mr(struct mlx5_vdpa_dev *mvdev,
+ struct mlx5_vdpa_mr *new_mr,
+ unsigned int asid)
{
- struct mlx5_vdpa_mr *mr = &mvdev->mr;
+ struct mlx5_vdpa_mr *old_mr = mvdev->mr[asid];
- mutex_lock(&mr->mkey_mtx);
+ mutex_lock(&mvdev->mr_mtx);
- _mlx5_vdpa_destroy_dvq_mr(mvdev, asid);
- _mlx5_vdpa_destroy_cvq_mr(mvdev, asid);
+ mvdev->mr[asid] = new_mr;
+ if (old_mr) {
+ _mlx5_vdpa_destroy_mr(mvdev, old_mr);
+ kfree(old_mr);
+ }
- mutex_unlock(&mr->mkey_mtx);
-}
+ mutex_unlock(&mvdev->mr_mtx);
-void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev)
-{
- mlx5_vdpa_destroy_mr_asid(mvdev, mvdev->group2asid[MLX5_VDPA_CVQ_GROUP]);
- mlx5_vdpa_destroy_mr_asid(mvdev, mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]);
}
-static int _mlx5_vdpa_create_cvq_mr(struct mlx5_vdpa_dev *mvdev,
- struct vhost_iotlb *iotlb,
- unsigned int asid)
+void mlx5_vdpa_destroy_mr_resources(struct mlx5_vdpa_dev *mvdev)
{
- if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
- return 0;
+ for (int i = 0; i < MLX5_VDPA_NUM_AS; i++)
+ mlx5_vdpa_destroy_mr(mvdev, mvdev->mr[i]);
- return dup_iotlb(mvdev, iotlb);
+ prune_iotlb(mvdev->cvq.iotlb);
}
-static int _mlx5_vdpa_create_dvq_mr(struct mlx5_vdpa_dev *mvdev,
- struct vhost_iotlb *iotlb,
- unsigned int asid)
+static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
+ struct mlx5_vdpa_mr *mr,
+ struct vhost_iotlb *iotlb)
{
- struct mlx5_vdpa_mr *mr = &mvdev->mr;
int err;
- if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] != asid)
- return 0;
-
- if (mr->initialized)
- return 0;
-
if (iotlb)
- err = create_user_mr(mvdev, iotlb);
+ err = create_user_mr(mvdev, mr, iotlb);
else
err = create_dma_mr(mvdev, mr);
if (err)
return err;
- mr->initialized = true;
+ mr->iotlb = vhost_iotlb_alloc(0, 0);
+ if (!mr->iotlb) {
+ err = -ENOMEM;
+ goto err_mr;
+ }
+
+ err = dup_iotlb(mr->iotlb, iotlb);
+ if (err)
+ goto err_iotlb;
return 0;
+
+err_iotlb:
+ vhost_iotlb_free(mr->iotlb);
+
+err_mr:
+ if (iotlb)
+ destroy_user_mr(mvdev, mr);
+ else
+ destroy_dma_mr(mvdev, mr);
+
+ return err;
}
-static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
- struct vhost_iotlb *iotlb, unsigned int asid)
+struct mlx5_vdpa_mr *mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
+ struct vhost_iotlb *iotlb)
{
+ struct mlx5_vdpa_mr *mr;
int err;
- err = _mlx5_vdpa_create_dvq_mr(mvdev, iotlb, asid);
- if (err)
- return err;
+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
+
+ mutex_lock(&mvdev->mr_mtx);
+ err = _mlx5_vdpa_create_mr(mvdev, mr, iotlb);
+ mutex_unlock(&mvdev->mr_mtx);
- err = _mlx5_vdpa_create_cvq_mr(mvdev, iotlb, asid);
if (err)
goto out_err;
- return 0;
+ return mr;
out_err:
- _mlx5_vdpa_destroy_dvq_mr(mvdev, asid);
-
- return err;
+ kfree(mr);
+ return ERR_PTR(err);
}
-int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,
- unsigned int asid)
+int mlx5_vdpa_update_cvq_iotlb(struct mlx5_vdpa_dev *mvdev,
+ struct vhost_iotlb *iotlb,
+ unsigned int asid)
{
int err;
- mutex_lock(&mvdev->mr.mkey_mtx);
- err = _mlx5_vdpa_create_mr(mvdev, iotlb, asid);
- mutex_unlock(&mvdev->mr.mkey_mtx);
+ if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
+ return 0;
+
+ spin_lock(&mvdev->cvq.iommu_lock);
+
+ prune_iotlb(mvdev->cvq.iotlb);
+ err = dup_iotlb(mvdev->cvq.iotlb, iotlb);
+
+ spin_unlock(&mvdev->cvq.iommu_lock);
+
return err;
}
-int mlx5_vdpa_handle_set_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,
- bool *change_map, unsigned int asid)
+int mlx5_vdpa_create_dma_mr(struct mlx5_vdpa_dev *mvdev)
{
- struct mlx5_vdpa_mr *mr = &mvdev->mr;
- int err = 0;
+ struct mlx5_vdpa_mr *mr;
+
+ mr = mlx5_vdpa_create_mr(mvdev, NULL);
+ if (IS_ERR(mr))
+ return PTR_ERR(mr);
- *change_map = false;
- mutex_lock(&mr->mkey_mtx);
- if (mr->initialized) {
- mlx5_vdpa_info(mvdev, "memory map update\n");
- *change_map = true;
+ mlx5_vdpa_update_mr(mvdev, mr, 0);
+
+ return mlx5_vdpa_update_cvq_iotlb(mvdev, NULL, 0);
+}
+
+int mlx5_vdpa_reset_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
+{
+ if (asid >= MLX5_VDPA_NUM_AS)
+ return -EINVAL;
+
+ mlx5_vdpa_destroy_mr(mvdev, mvdev->mr[asid]);
+
+ if (asid == 0 && MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
+ if (mlx5_vdpa_create_dma_mr(mvdev))
+ mlx5_vdpa_warn(mvdev, "create DMA MR failed\n");
+ } else {
+ mlx5_vdpa_update_cvq_iotlb(mvdev, NULL, asid);
}
- if (!*change_map)
- err = _mlx5_vdpa_create_mr(mvdev, iotlb, asid);
- mutex_unlock(&mr->mkey_mtx);
- return err;
+ return 0;
}
diff --git a/drivers/vdpa/mlx5/core/resources.c b/drivers/vdpa/mlx5/core/resources.c
index d5a59c9035..5c5a41b64b 100644
--- a/drivers/vdpa/mlx5/core/resources.c
+++ b/drivers/vdpa/mlx5/core/resources.c
@@ -256,7 +256,7 @@ int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev)
mlx5_vdpa_warn(mvdev, "resources already allocated\n");
return -EINVAL;
}
- mutex_init(&mvdev->mr.mkey_mtx);
+ mutex_init(&mvdev->mr_mtx);
res->uar = mlx5_get_uars_page(mdev);
if (IS_ERR(res->uar)) {
err = PTR_ERR(res->uar);
@@ -301,7 +301,7 @@ err_pd:
err_uctx:
mlx5_put_uars_page(mdev, res->uar);
err_uars:
- mutex_destroy(&mvdev->mr.mkey_mtx);
+ mutex_destroy(&mvdev->mr_mtx);
return err;
}
@@ -318,6 +318,6 @@ void mlx5_vdpa_free_resources(struct mlx5_vdpa_dev *mvdev)
dealloc_pd(mvdev, res->pdn, res->uid);
destroy_uctx(mvdev, res->uid);
mlx5_put_uars_page(mvdev->mdev, res->uar);
- mutex_destroy(&mvdev->mr.mkey_mtx);
+ mutex_destroy(&mvdev->mr_mtx);
res->valid = false;
}
diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c
index ca972af3c8..26ba7da6b4 100644
--- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
+++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
@@ -7,6 +7,7 @@
#include <uapi/linux/virtio_net.h>
#include <uapi/linux/virtio_ids.h>
#include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
#include <linux/virtio_config.h>
#include <linux/auxiliary_bus.h>
#include <linux/mlx5/cq.h>
@@ -861,6 +862,9 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
{
int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in);
u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {};
+ struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
+ struct mlx5_vdpa_mr *vq_mr;
+ struct mlx5_vdpa_mr *vq_desc_mr;
void *obj_context;
u16 mlx_features;
void *cmd_hdr;
@@ -913,7 +917,14 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
- MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey);
+ vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
+ if (vq_mr)
+ MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey);
+
+ vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
+ if (vq_desc_mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported))
+ MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, vq_desc_mr->mkey);
+
MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id);
MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size);
MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id);
@@ -2301,6 +2312,16 @@ static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx)
return MLX5_VDPA_DATAVQ_GROUP;
}
+static u32 mlx5_vdpa_get_vq_desc_group(struct vdpa_device *vdev, u16 idx)
+{
+ struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
+
+ if (is_ctrl_vq_idx(mvdev, idx))
+ return MLX5_VDPA_CVQ_GROUP;
+
+ return MLX5_VDPA_DATAVQ_DESC_GROUP;
+}
+
static u64 mlx_to_vritio_features(u16 dev_features)
{
u64 result = 0;
@@ -2539,6 +2560,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
flush_workqueue(ndev->mvdev.wq);
}
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -2673,7 +2699,8 @@ static void restore_channels_info(struct mlx5_vdpa_net *ndev)
}
static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
- struct vhost_iotlb *iotlb, unsigned int asid)
+ struct mlx5_vdpa_mr *new_mr,
+ unsigned int asid)
{
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
int err;
@@ -2681,28 +2708,21 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
suspend_vqs(ndev);
err = save_channels_info(ndev);
if (err)
- goto err_mr;
+ return err;
teardown_driver(ndev);
- mlx5_vdpa_destroy_mr_asid(mvdev, asid);
- err = mlx5_vdpa_create_mr(mvdev, iotlb, asid);
- if (err)
- goto err_mr;
+
+ mlx5_vdpa_update_mr(mvdev, new_mr, asid);
if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK) || mvdev->suspended)
- goto err_mr;
+ return 0;
restore_channels_info(ndev);
err = setup_driver(mvdev);
if (err)
- goto err_setup;
+ return err;
return 0;
-
-err_setup:
- mlx5_vdpa_destroy_mr_asid(mvdev, asid);
-err_mr:
- return err;
}
/* reslock must be held for this function */
@@ -2846,7 +2866,7 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
err_driver:
unregister_link_notifier(ndev);
err_setup:
- mlx5_vdpa_destroy_mr(&ndev->mvdev);
+ mlx5_vdpa_destroy_mr_resources(&ndev->mvdev);
ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED;
err_clear:
up_write(&ndev->reslock);
@@ -2861,7 +2881,7 @@ static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev)
mvdev->group2asid[i] = 0;
}
-static int mlx5_vdpa_reset(struct vdpa_device *vdev)
+static int mlx5_vdpa_compat_reset(struct vdpa_device *vdev, u32 flags)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
@@ -2873,7 +2893,8 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
unregister_link_notifier(ndev);
teardown_driver(ndev);
clear_vqs_ready(ndev);
- mlx5_vdpa_destroy_mr(&ndev->mvdev);
+ if (flags & VDPA_RESET_F_CLEAN_MAP)
+ mlx5_vdpa_destroy_mr_resources(&ndev->mvdev);
ndev->mvdev.status = 0;
ndev->mvdev.suspended = false;
ndev->cur_num_vqs = 0;
@@ -2884,8 +2905,9 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
init_group_to_asid_map(mvdev);
++mvdev->generation;
- if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
- if (mlx5_vdpa_create_mr(mvdev, NULL, 0))
+ if ((flags & VDPA_RESET_F_CLEAN_MAP) &&
+ MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
+ if (mlx5_vdpa_create_dma_mr(mvdev))
mlx5_vdpa_warn(mvdev, "create MR failed\n");
}
up_write(&ndev->reslock);
@@ -2893,6 +2915,11 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
return 0;
}
+static int mlx5_vdpa_reset(struct vdpa_device *vdev)
+{
+ return mlx5_vdpa_compat_reset(vdev, 0);
+}
+
static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev)
{
return sizeof(struct virtio_net_config);
@@ -2924,18 +2951,38 @@ static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev)
static int set_map_data(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,
unsigned int asid)
{
- bool change_map;
+ struct mlx5_vdpa_mr *new_mr;
int err;
- err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map, asid);
- if (err) {
- mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err);
- return err;
+ if (asid >= MLX5_VDPA_NUM_AS)
+ return -EINVAL;
+
+ if (vhost_iotlb_itree_first(iotlb, 0, U64_MAX)) {
+ new_mr = mlx5_vdpa_create_mr(mvdev, iotlb);
+ if (IS_ERR(new_mr)) {
+ err = PTR_ERR(new_mr);
+ mlx5_vdpa_warn(mvdev, "create map failed(%d)\n", err);
+ return err;
+ }
+ } else {
+ /* Empty iotlbs don't have an mr but will clear the previous mr. */
+ new_mr = NULL;
+ }
+
+ if (!mvdev->mr[asid]) {
+ mlx5_vdpa_update_mr(mvdev, new_mr, asid);
+ } else {
+ err = mlx5_vdpa_change_map(mvdev, new_mr, asid);
+ if (err) {
+ mlx5_vdpa_warn(mvdev, "change map failed(%d)\n", err);
+ goto out_err;
+ }
}
- if (change_map)
- err = mlx5_vdpa_change_map(mvdev, iotlb, asid);
+ return mlx5_vdpa_update_cvq_iotlb(mvdev, iotlb, asid);
+out_err:
+ mlx5_vdpa_destroy_mr(mvdev, new_mr);
return err;
}
@@ -2952,6 +2999,18 @@ static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid,
return err;
}
+static int mlx5_vdpa_reset_map(struct vdpa_device *vdev, unsigned int asid)
+{
+ struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
+ struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
+ int err;
+
+ down_write(&ndev->reslock);
+ err = mlx5_vdpa_reset_mr(mvdev, asid);
+ up_write(&ndev->reslock);
+ return err;
+}
+
static struct device *mlx5_get_vq_dma_dev(struct vdpa_device *vdev, u16 idx)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -2990,7 +3049,7 @@ static void mlx5_vdpa_free(struct vdpa_device *vdev)
ndev = to_mlx5_vdpa_ndev(mvdev);
free_resources(ndev);
- mlx5_vdpa_destroy_mr(mvdev);
+ mlx5_vdpa_destroy_mr_resources(mvdev);
if (!is_zero_ether_addr(ndev->config.mac)) {
pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
mlx5_mpfs_del_mac(pfmdev, ndev->config.mac);
@@ -3174,12 +3233,19 @@ static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group,
unsigned int asid)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
+ int err = 0;
if (group >= MLX5_VDPA_NUMVQ_GROUPS)
return -EINVAL;
mvdev->group2asid[group] = asid;
- return 0;
+
+ mutex_lock(&mvdev->mr_mtx);
+ if (group == MLX5_VDPA_CVQ_GROUP && mvdev->mr[asid])
+ err = mlx5_vdpa_update_cvq_iotlb(mvdev, mvdev->mr[asid]->iotlb, asid);
+ mutex_unlock(&mvdev->mr_mtx);
+
+ return err;
}
static const struct vdpa_config_ops mlx5_vdpa_ops = {
@@ -3196,7 +3262,9 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
.get_vq_irq = mlx5_get_vq_irq,
.get_vq_align = mlx5_vdpa_get_vq_align,
.get_vq_group = mlx5_vdpa_get_vq_group,
+ .get_vq_desc_group = mlx5_vdpa_get_vq_desc_group, /* Op disabled if not supported. */
.get_device_features = mlx5_vdpa_get_device_features,
+ .get_backend_features = mlx5_vdpa_get_backend_features,
.set_driver_features = mlx5_vdpa_set_driver_features,
.get_driver_features = mlx5_vdpa_get_driver_features,
.set_config_cb = mlx5_vdpa_set_config_cb,
@@ -3206,11 +3274,13 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
.get_status = mlx5_vdpa_get_status,
.set_status = mlx5_vdpa_set_status,
.reset = mlx5_vdpa_reset,
+ .compat_reset = mlx5_vdpa_compat_reset,
.get_config_size = mlx5_vdpa_get_config_size,
.get_config = mlx5_vdpa_get_config,
.set_config = mlx5_vdpa_set_config,
.get_generation = mlx5_vdpa_get_generation,
.set_map = mlx5_vdpa_set_map,
+ .reset_map = mlx5_vdpa_reset_map,
.set_group_asid = mlx5_set_group_asid,
.get_vq_dma_dev = mlx5_get_vq_dma_dev,
.free = mlx5_vdpa_free,
@@ -3294,6 +3364,7 @@ struct mlx5_vdpa_mgmtdev {
struct vdpa_mgmt_dev mgtdev;
struct mlx5_adev *madev;
struct mlx5_vdpa_net *ndev;
+ struct vdpa_config_ops vdpa_ops;
};
static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu)
@@ -3407,7 +3478,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
max_vqs = 2;
}
- ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops,
+ ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mgtdev->vdpa_ops,
MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false);
if (IS_ERR(ndev))
return PTR_ERR(ndev);
@@ -3490,7 +3561,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
goto err_mpfs;
if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
- err = mlx5_vdpa_create_mr(mvdev, NULL, 0);
+ err = mlx5_vdpa_create_dma_mr(mvdev);
if (err)
goto err_res;
}
@@ -3520,7 +3591,7 @@ err_reg:
err_res2:
free_resources(ndev);
err_mr:
- mlx5_vdpa_destroy_mr(mvdev);
+ mlx5_vdpa_destroy_mr_resources(mvdev);
err_res:
mlx5_vdpa_free_resources(&ndev->mvdev);
err_mpfs:
@@ -3580,6 +3651,10 @@ static int mlx5v_probe(struct auxiliary_device *adev,
MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1;
mgtdev->mgtdev.supported_features = get_supported_features(mdev);
mgtdev->madev = madev;
+ mgtdev->vdpa_ops = mlx5_vdpa_ops;
+
+ if (!MLX5_CAP_DEV_VDPA_EMULATION(mdev, desc_group_mkey_supported))
+ mgtdev->vdpa_ops.get_vq_desc_group = NULL;
err = vdpa_mgmtdev_register(&mgtdev->mgtdev);
if (err)
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index 76d41058ad..be2925d0d2 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -139,7 +139,7 @@ static void vdpasim_vq_reset(struct vdpasim *vdpasim,
vq->vring.notify = NULL;
}
-static void vdpasim_do_reset(struct vdpasim *vdpasim)
+static void vdpasim_do_reset(struct vdpasim *vdpasim, u32 flags)
{
int i;
@@ -151,11 +151,13 @@ static void vdpasim_do_reset(struct vdpasim *vdpasim)
&vdpasim->iommu_lock);
}
- for (i = 0; i < vdpasim->dev_attr.nas; i++) {
- vhost_iotlb_reset(&vdpasim->iommu[i]);
- vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX,
- 0, VHOST_MAP_RW);
- vdpasim->iommu_pt[i] = true;
+ if (flags & VDPA_RESET_F_CLEAN_MAP) {
+ for (i = 0; i < vdpasim->dev_attr.nas; i++) {
+ vhost_iotlb_reset(&vdpasim->iommu[i]);
+ vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX,
+ 0, VHOST_MAP_RW);
+ vdpasim->iommu_pt[i] = true;
+ }
}
vdpasim->running = true;
@@ -259,8 +261,12 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
if (!vdpasim->iommu_pt)
goto err_iommu;
- for (i = 0; i < vdpasim->dev_attr.nas; i++)
+ for (i = 0; i < vdpasim->dev_attr.nas; i++) {
vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);
+ vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 0,
+ VHOST_MAP_RW);
+ vdpasim->iommu_pt[i] = true;
+ }
for (i = 0; i < dev_attr->nvqs; i++)
vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],
@@ -480,18 +486,23 @@ static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status)
mutex_unlock(&vdpasim->mutex);
}
-static int vdpasim_reset(struct vdpa_device *vdpa)
+static int vdpasim_compat_reset(struct vdpa_device *vdpa, u32 flags)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
mutex_lock(&vdpasim->mutex);
vdpasim->status = 0;
- vdpasim_do_reset(vdpasim);
+ vdpasim_do_reset(vdpasim, flags);
mutex_unlock(&vdpasim->mutex);
return 0;
}
+static int vdpasim_reset(struct vdpa_device *vdpa)
+{
+ return vdpasim_compat_reset(vdpa, 0);
+}
+
static int vdpasim_suspend(struct vdpa_device *vdpa)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
@@ -637,6 +648,25 @@ err:
return ret;
}
+static int vdpasim_reset_map(struct vdpa_device *vdpa, unsigned int asid)
+{
+ struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+ if (asid >= vdpasim->dev_attr.nas)
+ return -EINVAL;
+
+ spin_lock(&vdpasim->iommu_lock);
+ if (vdpasim->iommu_pt[asid])
+ goto out;
+ vhost_iotlb_reset(&vdpasim->iommu[asid]);
+ vhost_iotlb_add_range(&vdpasim->iommu[asid], 0, ULONG_MAX,
+ 0, VHOST_MAP_RW);
+ vdpasim->iommu_pt[asid] = true;
+out:
+ spin_unlock(&vdpasim->iommu_lock);
+ return 0;
+}
+
static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm)
{
struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
@@ -749,6 +779,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
.get_status = vdpasim_get_status,
.set_status = vdpasim_set_status,
.reset = vdpasim_reset,
+ .compat_reset = vdpasim_compat_reset,
.suspend = vdpasim_suspend,
.resume = vdpasim_resume,
.get_config_size = vdpasim_get_config_size,
@@ -759,6 +790,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
.set_group_asid = vdpasim_set_group_asid,
.dma_map = vdpasim_dma_map,
.dma_unmap = vdpasim_dma_unmap,
+ .reset_map = vdpasim_reset_map,
.bind_mm = vdpasim_bind_mm,
.unbind_mm = vdpasim_unbind_mm,
.free = vdpasim_free,
@@ -787,6 +819,7 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
.get_status = vdpasim_get_status,
.set_status = vdpasim_set_status,
.reset = vdpasim_reset,
+ .compat_reset = vdpasim_compat_reset,
.suspend = vdpasim_suspend,
.resume = vdpasim_resume,
.get_config_size = vdpasim_get_config_size,
@@ -796,6 +829,7 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
.get_iova_range = vdpasim_get_iova_range,
.set_group_asid = vdpasim_set_group_asid,
.set_map = vdpasim_set_map,
+ .reset_map = vdpasim_reset_map,
.bind_mm = vdpasim_bind_mm,
.unbind_mm = vdpasim_unbind_mm,
.free = vdpasim_free,
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index df7869537e..0ddd4b8abe 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -134,7 +134,6 @@ static DEFINE_MUTEX(vduse_lock);
static DEFINE_IDR(vduse_idr);
static dev_t vduse_major;
-static struct class *vduse_class;
static struct cdev vduse_ctrl_cdev;
static struct cdev vduse_cdev;
static struct workqueue_struct *vduse_irq_wq;
@@ -1528,6 +1527,16 @@ static const struct kobj_type vq_type = {
.default_groups = vq_groups,
};
+static char *vduse_devnode(const struct device *dev, umode_t *mode)
+{
+ return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
+}
+
+static const struct class vduse_class = {
+ .name = "vduse",
+ .devnode = vduse_devnode,
+};
+
static void vduse_dev_deinit_vqs(struct vduse_dev *dev)
{
int i;
@@ -1638,7 +1647,7 @@ static int vduse_destroy_dev(char *name)
mutex_unlock(&dev->lock);
vduse_dev_reset(dev);
- device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
+ device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
idr_remove(&vduse_idr, dev->minor);
kvfree(dev->config);
vduse_dev_deinit_vqs(dev);
@@ -1805,7 +1814,7 @@ static int vduse_create_dev(struct vduse_dev_config *config,
dev->minor = ret;
dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
- dev->dev = device_create_with_groups(vduse_class, NULL,
+ dev->dev = device_create_with_groups(&vduse_class, NULL,
MKDEV(MAJOR(vduse_major), dev->minor),
dev, vduse_dev_groups, "%s", config->name);
if (IS_ERR(dev->dev)) {
@@ -1821,7 +1830,7 @@ static int vduse_create_dev(struct vduse_dev_config *config,
return 0;
err_vqs:
- device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
+ device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
err_dev:
idr_remove(&vduse_idr, dev->minor);
err_idr:
@@ -1934,11 +1943,6 @@ static const struct file_operations vduse_ctrl_fops = {
.llseek = noop_llseek,
};
-static char *vduse_devnode(const struct device *dev, umode_t *mode)
-{
- return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
-}
-
struct vduse_mgmt_dev {
struct vdpa_mgmt_dev mgmt_dev;
struct device dev;
@@ -2082,11 +2086,9 @@ static int vduse_init(void)
int ret;
struct device *dev;
- vduse_class = class_create("vduse");
- if (IS_ERR(vduse_class))
- return PTR_ERR(vduse_class);
-
- vduse_class->devnode = vduse_devnode;
+ ret = class_register(&vduse_class);
+ if (ret)
+ return ret;
ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
if (ret)
@@ -2099,7 +2101,7 @@ static int vduse_init(void)
if (ret)
goto err_ctrl_cdev;
- dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
+ dev = device_create(&vduse_class, NULL, vduse_major, NULL, "control");
if (IS_ERR(dev)) {
ret = PTR_ERR(dev);
goto err_device;
@@ -2141,13 +2143,13 @@ err_bound_wq:
err_wq:
cdev_del(&vduse_cdev);
err_cdev:
- device_destroy(vduse_class, vduse_major);
+ device_destroy(&vduse_class, vduse_major);
err_device:
cdev_del(&vduse_ctrl_cdev);
err_ctrl_cdev:
unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
err_chardev_region:
- class_destroy(vduse_class);
+ class_unregister(&vduse_class);
return ret;
}
module_init(vduse_init);
@@ -2159,10 +2161,10 @@ static void vduse_exit(void)
destroy_workqueue(vduse_irq_bound_wq);
destroy_workqueue(vduse_irq_wq);
cdev_del(&vduse_cdev);
- device_destroy(vduse_class, vduse_major);
+ device_destroy(&vduse_class, vduse_major);
cdev_del(&vduse_ctrl_cdev);
unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
- class_destroy(vduse_class);
+ class_unregister(&vduse_class);
}
module_exit(vduse_exit);
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-29 11:14:19 +0000