summaryrefslogtreecommitdiffstats
path: root/drivers/infiniband/hw/erdma/erdma_verbs.h
blob: 4f02ba06b210a4f967ca9e78c2153c9ac66a9d96 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */

/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
/*          Kai Shen <kaishen@linux.alibaba.com> */
/* Copyright (c) 2020-2022, Alibaba Group. */

#ifndef __ERDMA_VERBS_H__
#define __ERDMA_VERBS_H__

#include "erdma.h"

/* RDMA Capability. */
#define ERDMA_MAX_PD (128 * 1024)
#define ERDMA_MAX_SEND_WR 8192
#define ERDMA_MAX_ORD 128
#define ERDMA_MAX_IRD 128
#define ERDMA_MAX_SGE_RD 1
#define ERDMA_MAX_CONTEXT (128 * 1024)
#define ERDMA_MAX_SEND_SGE 6
#define ERDMA_MAX_RECV_SGE 1
#define ERDMA_MAX_INLINE (sizeof(struct erdma_sge) * (ERDMA_MAX_SEND_SGE))
#define ERDMA_MAX_FRMR_PA 512

enum {
	ERDMA_MMAP_IO_NC = 0, /* no cache */
};

struct erdma_user_mmap_entry {
	struct rdma_user_mmap_entry rdma_entry;
	u64 address;
	u8 mmap_flag;
};

struct erdma_ext_db_info {
	bool enable;
	u16 sdb_off;
	u16 rdb_off;
	u16 cdb_off;
};

struct erdma_ucontext {
	struct ib_ucontext ibucontext;

	struct erdma_ext_db_info ext_db;

	u64 sdb;
	u64 rdb;
	u64 cdb;

	struct rdma_user_mmap_entry *sq_db_mmap_entry;
	struct rdma_user_mmap_entry *rq_db_mmap_entry;
	struct rdma_user_mmap_entry *cq_db_mmap_entry;

	/* doorbell records */
	struct list_head dbrecords_page_list;
	struct mutex dbrecords_page_mutex;
};

struct erdma_pd {
	struct ib_pd ibpd;
	u32 pdn;
};

/*
 * MemoryRegion definition.
 */
#define ERDMA_MAX_INLINE_MTT_ENTRIES 4
#define MTT_SIZE(mtt_cnt) ((mtt_cnt) << 3) /* per mtt entry takes 8 Bytes. */
#define ERDMA_MR_MAX_MTT_CNT 524288
#define ERDMA_MTT_ENTRY_SIZE 8

#define ERDMA_MR_TYPE_NORMAL 0
#define ERDMA_MR_TYPE_FRMR 1
#define ERDMA_MR_TYPE_DMA 2

#define ERDMA_MR_MTT_0LEVEL 0
#define ERDMA_MR_MTT_1LEVEL 1

#define ERDMA_MR_ACC_RA BIT(0)
#define ERDMA_MR_ACC_LR BIT(1)
#define ERDMA_MR_ACC_LW BIT(2)
#define ERDMA_MR_ACC_RR BIT(3)
#define ERDMA_MR_ACC_RW BIT(4)

static inline u8 to_erdma_access_flags(int access)
{
	return (access & IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) |
	       (access & IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) |
	       (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0) |
	       (access & IB_ACCESS_REMOTE_ATOMIC ? ERDMA_MR_ACC_RA : 0);
}

/* Hierarchical storage structure for MTT entries */
struct erdma_mtt {
	u64 *buf;
	size_t size;

	bool continuous;
	union {
		dma_addr_t buf_dma;
		struct {
			struct scatterlist *sglist;
			u32 nsg;
			u32 level;
		};
	};

	struct erdma_mtt *low_level;
};

struct erdma_mem {
	struct ib_umem *umem;
	struct erdma_mtt *mtt;

	u32 page_size;
	u32 page_offset;
	u32 page_cnt;
	u32 mtt_nents;

	u64 va;
	u64 len;
};

struct erdma_mr {
	struct ib_mr ibmr;
	struct erdma_mem mem;
	u8 type;
	u8 access;
	u8 valid;
};

struct erdma_user_dbrecords_page {
	struct list_head list;
	struct ib_umem *umem;
	u64 va;
	int refcnt;
};

struct erdma_uqp {
	struct erdma_mem sq_mem;
	struct erdma_mem rq_mem;

	dma_addr_t sq_dbrec_dma;
	dma_addr_t rq_dbrec_dma;

	struct erdma_user_dbrecords_page *user_dbr_page;

	u32 rq_offset;
};

struct erdma_kqp {
	u16 sq_pi;
	u16 sq_ci;

	u16 rq_pi;
	u16 rq_ci;

	u64 *swr_tbl;
	u64 *rwr_tbl;

	void __iomem *hw_sq_db;
	void __iomem *hw_rq_db;

	void *sq_buf;
	dma_addr_t sq_buf_dma_addr;

	void *rq_buf;
	dma_addr_t rq_buf_dma_addr;

	void *sq_dbrec;
	void *rq_dbrec;

	dma_addr_t sq_dbrec_dma;
	dma_addr_t rq_dbrec_dma;

	u8 sig_all;
};

enum erdma_qp_state {
	ERDMA_QP_STATE_IDLE = 0,
	ERDMA_QP_STATE_RTR = 1,
	ERDMA_QP_STATE_RTS = 2,
	ERDMA_QP_STATE_CLOSING = 3,
	ERDMA_QP_STATE_TERMINATE = 4,
	ERDMA_QP_STATE_ERROR = 5,
	ERDMA_QP_STATE_UNDEF = 7,
	ERDMA_QP_STATE_COUNT = 8
};

enum erdma_qp_attr_mask {
	ERDMA_QP_ATTR_STATE = (1 << 0),
	ERDMA_QP_ATTR_LLP_HANDLE = (1 << 2),
	ERDMA_QP_ATTR_ORD = (1 << 3),
	ERDMA_QP_ATTR_IRD = (1 << 4),
	ERDMA_QP_ATTR_SQ_SIZE = (1 << 5),
	ERDMA_QP_ATTR_RQ_SIZE = (1 << 6),
	ERDMA_QP_ATTR_MPA = (1 << 7)
};

enum erdma_qp_flags {
	ERDMA_QP_IN_FLUSHING = (1 << 0),
};

struct erdma_qp_attrs {
	enum erdma_qp_state state;
	enum erdma_cc_alg cc; /* Congestion control algorithm */
	u32 sq_size;
	u32 rq_size;
	u32 orq_size;
	u32 irq_size;
	u32 max_send_sge;
	u32 max_recv_sge;
	u32 cookie;
#define ERDMA_QP_ACTIVE 0
#define ERDMA_QP_PASSIVE 1
	u8 qp_type;
	u8 pd_len;
};

struct erdma_qp {
	struct ib_qp ibqp;
	struct kref ref;
	struct completion safe_free;
	struct erdma_dev *dev;
	struct erdma_cep *cep;
	struct rw_semaphore state_lock;

	unsigned long flags;
	struct delayed_work reflush_dwork;

	union {
		struct erdma_kqp kern_qp;
		struct erdma_uqp user_qp;
	};

	struct erdma_cq *scq;
	struct erdma_cq *rcq;

	struct erdma_qp_attrs attrs;
	spinlock_t lock;
};

struct erdma_kcq_info {
	void *qbuf;
	dma_addr_t qbuf_dma_addr;
	u32 ci;
	u32 cmdsn;
	u32 notify_cnt;

	spinlock_t lock;
	u8 __iomem *db;
	u64 *dbrec;
	dma_addr_t dbrec_dma;
};

struct erdma_ucq_info {
	struct erdma_mem qbuf_mem;
	struct erdma_user_dbrecords_page *user_dbr_page;
	dma_addr_t dbrec_dma;
};

struct erdma_cq {
	struct ib_cq ibcq;
	u32 cqn;

	u32 depth;
	u32 assoc_eqn;

	union {
		struct erdma_kcq_info kern_cq;
		struct erdma_ucq_info user_cq;
	};
};

#define QP_ID(qp) ((qp)->ibqp.qp_num)

static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id)
{
	return (struct erdma_qp *)xa_load(&dev->qp_xa, id);
}

static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id)
{
	return (struct erdma_cq *)xa_load(&dev->cq_xa, id);
}

void erdma_qp_get(struct erdma_qp *qp);
void erdma_qp_put(struct erdma_qp *qp);
int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
			     enum erdma_qp_attr_mask mask);
void erdma_qp_llp_close(struct erdma_qp *qp);
void erdma_qp_cm_drop(struct erdma_qp *qp);

static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx)
{
	return container_of(ibctx, struct erdma_ucontext, ibucontext);
}

static inline struct erdma_pd *to_epd(struct ib_pd *pd)
{
	return container_of(pd, struct erdma_pd, ibpd);
}

static inline struct erdma_mr *to_emr(struct ib_mr *ibmr)
{
	return container_of(ibmr, struct erdma_mr, ibmr);
}

static inline struct erdma_qp *to_eqp(struct ib_qp *qp)
{
	return container_of(qp, struct erdma_qp, ibqp);
}

static inline struct erdma_cq *to_ecq(struct ib_cq *ibcq)
{
	return container_of(ibcq, struct erdma_cq, ibcq);
}

static inline struct erdma_user_mmap_entry *
to_emmap(struct rdma_user_mmap_entry *ibmmap)
{
	return container_of(ibmmap, struct erdma_user_mmap_entry, rdma_entry);
}

int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *data);
void erdma_dealloc_ucontext(struct ib_ucontext *ibctx);
int erdma_query_device(struct ib_device *dev, struct ib_device_attr *attr,
		       struct ib_udata *data);
int erdma_get_port_immutable(struct ib_device *dev, u32 port,
			     struct ib_port_immutable *ib_port_immutable);
int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
		    struct ib_udata *data);
int erdma_query_port(struct ib_device *dev, u32 port,
		     struct ib_port_attr *attr);
int erdma_query_gid(struct ib_device *dev, u32 port, int idx,
		    union ib_gid *gid);
int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *data);
int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
		    struct ib_udata *data);
int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
		   struct ib_qp_init_attr *init_attr);
int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
		    struct ib_udata *data);
int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
				u64 virt, int access, struct ib_udata *udata);
struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);
int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data);
int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);
void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
void erdma_qp_get_ref(struct ib_qp *ibqp);
void erdma_qp_put_ref(struct ib_qp *ibqp);
struct ib_qp *erdma_get_ibqp(struct ib_device *dev, int id);
int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,
		    const struct ib_send_wr **bad_send_wr);
int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
		    const struct ib_recv_wr **bad_recv_wr);
int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
				u32 max_num_sg);
int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
		    unsigned int *sg_offset);
void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason);
void erdma_set_mtu(struct erdma_dev *dev, u32 mtu);
struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
						u32 port_num);
int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
		       u32 port, int index);

#endif