summaryrefslogtreecommitdiffstats
path: root/drivers/scsi/cxgbi
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/cxgbi')
-rw-r--r--drivers/scsi/cxgbi/Kconfig3
-rw-r--r--drivers/scsi/cxgbi/Makefile5
-rw-r--r--drivers/scsi/cxgbi/cxgb3i/Kbuild5
-rw-r--r--drivers/scsi/cxgbi/cxgb3i/Kconfig12
-rw-r--r--drivers/scsi/cxgbi/cxgb3i/cxgb3i.c1414
-rw-r--r--drivers/scsi/cxgbi/cxgb3i/cxgb3i.h62
-rw-r--r--drivers/scsi/cxgbi/cxgb4i/Kbuild5
-rw-r--r--drivers/scsi/cxgbi/cxgb4i/Kconfig13
-rw-r--r--drivers/scsi/cxgbi/cxgb4i/cxgb4i.c2472
-rw-r--r--drivers/scsi/cxgbi/cxgb4i/cxgb4i.h28
-rw-r--r--drivers/scsi/cxgbi/libcxgbi.c3098
-rw-r--r--drivers/scsi/cxgbi/libcxgbi.h647
12 files changed, 7764 insertions, 0 deletions
diff --git a/drivers/scsi/cxgbi/Kconfig b/drivers/scsi/cxgbi/Kconfig
new file mode 100644
index 000000000..75f9428a8
--- /dev/null
+++ b/drivers/scsi/cxgbi/Kconfig
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-only
+source "drivers/scsi/cxgbi/cxgb3i/Kconfig"
+source "drivers/scsi/cxgbi/cxgb4i/Kconfig"
diff --git a/drivers/scsi/cxgbi/Makefile b/drivers/scsi/cxgbi/Makefile
new file mode 100644
index 000000000..abfd38a26
--- /dev/null
+++ b/drivers/scsi/cxgbi/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+ccflags-y += -I $(srctree)/drivers/net/ethernet/chelsio/libcxgb
+
+obj-$(CONFIG_SCSI_CXGB3_ISCSI) += libcxgbi.o cxgb3i/
+obj-$(CONFIG_SCSI_CXGB4_ISCSI) += libcxgbi.o cxgb4i/
diff --git a/drivers/scsi/cxgbi/cxgb3i/Kbuild b/drivers/scsi/cxgbi/cxgb3i/Kbuild
new file mode 100644
index 000000000..8d8a43f5e
--- /dev/null
+++ b/drivers/scsi/cxgbi/cxgb3i/Kbuild
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+ccflags-y += -I$(srctree)/drivers/net/ethernet/chelsio/cxgb3
+ccflags-y += -I$(srctree)/drivers/net/ethernet/chelsio/libcxgb
+
+obj-$(CONFIG_SCSI_CXGB3_ISCSI) += cxgb3i.o
diff --git a/drivers/scsi/cxgbi/cxgb3i/Kconfig b/drivers/scsi/cxgbi/cxgb3i/Kconfig
new file mode 100644
index 000000000..e20e6f3bf
--- /dev/null
+++ b/drivers/scsi/cxgbi/cxgb3i/Kconfig
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config SCSI_CXGB3_ISCSI
+ tristate "Chelsio T3 iSCSI support"
+ depends on PCI && INET && (IPV6 || IPV6=n)
+ select NETDEVICES
+ select ETHERNET
+ select NET_VENDOR_CHELSIO
+ select CHELSIO_T3
+ select CHELSIO_LIB
+ select SCSI_ISCSI_ATTRS
+ help
+ This driver supports iSCSI offload for the Chelsio T3 devices.
diff --git a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
new file mode 100644
index 000000000..edcd3fab6
--- /dev/null
+++ b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
@@ -0,0 +1,1414 @@
+/*
+ * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
+ *
+ * Copyright (C) 2003-2015 Chelsio Communications. All rights reserved.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
+ * release for licensing terms and conditions.
+ *
+ * Written by: Dimitris Michailidis (dm@chelsio.com)
+ * Karen Xie (kxie@chelsio.com)
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <scsi/scsi_host.h>
+
+#include "common.h"
+#include "t3_cpl.h"
+#include "t3cdev.h"
+#include "cxgb3_defs.h"
+#include "cxgb3_ctl_defs.h"
+#include "cxgb3_offload.h"
+#include "firmware_exports.h"
+#include "cxgb3i.h"
+
+static unsigned int dbg_level;
+#include "../libcxgbi.h"
+
+#define DRV_MODULE_NAME "cxgb3i"
+#define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver"
+#define DRV_MODULE_VERSION "2.0.1-ko"
+#define DRV_MODULE_RELDATE "Apr. 2015"
+
+static char version[] =
+ DRV_MODULE_DESC " " DRV_MODULE_NAME
+ " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
+
+MODULE_AUTHOR("Chelsio Communications, Inc.");
+MODULE_DESCRIPTION(DRV_MODULE_DESC);
+MODULE_VERSION(DRV_MODULE_VERSION);
+MODULE_LICENSE("GPL");
+
+module_param(dbg_level, uint, 0644);
+MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");
+
+static int cxgb3i_rcv_win = 256 * 1024;
+module_param(cxgb3i_rcv_win, int, 0644);
+MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");
+
+static int cxgb3i_snd_win = 128 * 1024;
+module_param(cxgb3i_snd_win, int, 0644);
+MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");
+
+static int cxgb3i_rx_credit_thres = 10 * 1024;
+module_param(cxgb3i_rx_credit_thres, int, 0644);
+MODULE_PARM_DESC(cxgb3i_rx_credit_thres,
+ "RX credits return threshold in bytes (default=10KB)");
+
+static unsigned int cxgb3i_max_connect = 8 * 1024;
+module_param(cxgb3i_max_connect, uint, 0644);
+MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");
+
+static unsigned int cxgb3i_sport_base = 20000;
+module_param(cxgb3i_sport_base, uint, 0644);
+MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");
+
+static void cxgb3i_dev_open(struct t3cdev *);
+static void cxgb3i_dev_close(struct t3cdev *);
+static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);
+
+static struct cxgb3_client t3_client = {
+ .name = DRV_MODULE_NAME,
+ .handlers = cxgb3i_cpl_handlers,
+ .add = cxgb3i_dev_open,
+ .remove = cxgb3i_dev_close,
+ .event_handler = cxgb3i_dev_event_handler,
+};
+
+static struct scsi_host_template cxgb3i_host_template = {
+ .module = THIS_MODULE,
+ .name = DRV_MODULE_NAME,
+ .proc_name = DRV_MODULE_NAME,
+ .can_queue = CXGB3I_SCSI_HOST_QDEPTH,
+ .queuecommand = iscsi_queuecommand,
+ .change_queue_depth = scsi_change_queue_depth,
+ .sg_tablesize = SG_ALL,
+ .max_sectors = 0xFFFF,
+ .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
+ .eh_timed_out = iscsi_eh_cmd_timed_out,
+ .eh_abort_handler = iscsi_eh_abort,
+ .eh_device_reset_handler = iscsi_eh_device_reset,
+ .eh_target_reset_handler = iscsi_eh_recover_target,
+ .target_alloc = iscsi_target_alloc,
+ .dma_boundary = PAGE_SIZE - 1,
+ .this_id = -1,
+ .track_queue_depth = 1,
+};
+
+static struct iscsi_transport cxgb3i_iscsi_transport = {
+ .owner = THIS_MODULE,
+ .name = DRV_MODULE_NAME,
+ /* owner and name should be set already */
+ .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
+ | CAP_DATADGST | CAP_DIGEST_OFFLOAD |
+ CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
+ .attr_is_visible = cxgbi_attr_is_visible,
+ .get_host_param = cxgbi_get_host_param,
+ .set_host_param = cxgbi_set_host_param,
+ /* session management */
+ .create_session = cxgbi_create_session,
+ .destroy_session = cxgbi_destroy_session,
+ .get_session_param = iscsi_session_get_param,
+ /* connection management */
+ .create_conn = cxgbi_create_conn,
+ .bind_conn = cxgbi_bind_conn,
+ .unbind_conn = iscsi_conn_unbind,
+ .destroy_conn = iscsi_tcp_conn_teardown,
+ .start_conn = iscsi_conn_start,
+ .stop_conn = iscsi_conn_stop,
+ .get_conn_param = iscsi_conn_get_param,
+ .set_param = cxgbi_set_conn_param,
+ .get_stats = cxgbi_get_conn_stats,
+ /* pdu xmit req from user space */
+ .send_pdu = iscsi_conn_send_pdu,
+ /* task */
+ .init_task = iscsi_tcp_task_init,
+ .xmit_task = iscsi_tcp_task_xmit,
+ .cleanup_task = cxgbi_cleanup_task,
+ /* pdu */
+ .alloc_pdu = cxgbi_conn_alloc_pdu,
+ .init_pdu = cxgbi_conn_init_pdu,
+ .xmit_pdu = cxgbi_conn_xmit_pdu,
+ .parse_pdu_itt = cxgbi_parse_pdu_itt,
+ /* TCP connect/disconnect */
+ .get_ep_param = cxgbi_get_ep_param,
+ .ep_connect = cxgbi_ep_connect,
+ .ep_poll = cxgbi_ep_poll,
+ .ep_disconnect = cxgbi_ep_disconnect,
+ /* Error recovery timeout call */
+ .session_recovery_timedout = iscsi_session_recovery_timedout,
+};
+
+static struct scsi_transport_template *cxgb3i_stt;
+
+/*
+ * CPL (Chelsio Protocol Language) defines a message passing interface between
+ * the host driver and Chelsio asic.
+ * The section below implments CPLs that related to iscsi tcp connection
+ * open/close/abort and data send/receive.
+ */
+
+static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);
+
+static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
+ const struct l2t_entry *e)
+{
+ unsigned int wscale = cxgbi_sock_compute_wscale(csk->rcv_win);
+ struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;
+
+ skb->priority = CPL_PRIORITY_SETUP;
+
+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));
+ req->local_port = csk->saddr.sin_port;
+ req->peer_port = csk->daddr.sin_port;
+ req->local_ip = csk->saddr.sin_addr.s_addr;
+ req->peer_ip = csk->daddr.sin_addr.s_addr;
+
+ req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |
+ V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |
+ V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
+ req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |
+ V_RCV_BUFSIZ(csk->rcv_win >> 10));
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
+ csk, csk->state, csk->flags, csk->atid,
+ &req->local_ip, ntohs(req->local_port),
+ &req->peer_ip, ntohs(req->peer_port),
+ csk->mss_idx, e->idx, e->smt_idx);
+
+ l2t_send(csk->cdev->lldev, skb, csk->l2t);
+}
+
+static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
+{
+ cxgbi_sock_act_open_req_arp_failure(NULL, skb);
+}
+
+/*
+ * CPL connection close request: host ->
+ *
+ * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
+ * the write queue (i.e., after any unsent txt data).
+ */
+static void send_close_req(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb = csk->cpl_close;
+ struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
+ unsigned int tid = csk->tid;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ csk->cpl_close = NULL;
+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
+ req->wr.wr_lo = htonl(V_WR_TID(tid));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
+ req->rsvd = htonl(csk->write_seq);
+
+ cxgbi_sock_skb_entail(csk, skb);
+ if (csk->state >= CTP_ESTABLISHED)
+ push_tx_frames(csk, 1);
+}
+
+/*
+ * CPL connection abort request: host ->
+ *
+ * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
+ * for the same connection and also that we do not try to send a message
+ * after the connection has closed.
+ */
+static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)
+{
+ struct cpl_abort_req *req = cplhdr(skb);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "t3dev 0x%p, tid %u, skb 0x%p.\n",
+ tdev, GET_TID(req), skb);
+ req->cmd = CPL_ABORT_NO_RST;
+ cxgb3_ofld_send(tdev, skb);
+}
+
+static void send_abort_req(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb = csk->cpl_abort_req;
+ struct cpl_abort_req *req;
+
+ if (unlikely(csk->state == CTP_ABORTING || !skb))
+ return;
+ cxgbi_sock_set_state(csk, CTP_ABORTING);
+ cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
+ /* Purge the send queue so we don't send anything after an abort. */
+ cxgbi_sock_purge_write_queue(csk);
+
+ csk->cpl_abort_req = NULL;
+ req = (struct cpl_abort_req *)skb->head;
+ skb->priority = CPL_PRIORITY_DATA;
+ set_arp_failure_handler(skb, abort_arp_failure);
+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
+ req->wr.wr_lo = htonl(V_WR_TID(csk->tid));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
+ req->rsvd0 = htonl(csk->snd_nxt);
+ req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
+ req->cmd = CPL_ABORT_SEND_RST;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
+ csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
+ req->rsvd1);
+
+ l2t_send(csk->cdev->lldev, skb, csk->l2t);
+}
+
+/*
+ * CPL connection abort reply: host ->
+ *
+ * Send an ABORT_RPL message in response of the ABORT_REQ received.
+ */
+static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
+{
+ struct sk_buff *skb = csk->cpl_abort_rpl;
+ struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, status %d.\n",
+ csk, csk->state, csk->flags, csk->tid, rst_status);
+
+ csk->cpl_abort_rpl = NULL;
+ skb->priority = CPL_PRIORITY_DATA;
+ rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
+ rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));
+ OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
+ rpl->cmd = rst_status;
+ cxgb3_ofld_send(csk->cdev->lldev, skb);
+}
+
+/*
+ * CPL connection rx data ack: host ->
+ * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
+ * credits sent.
+ */
+static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
+{
+ struct sk_buff *skb;
+ struct cpl_rx_data_ack *req;
+ u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
+ csk, csk->state, csk->flags, csk->tid, credits, dack);
+
+ skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
+ if (!skb) {
+ pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
+ return 0;
+ }
+ req = (struct cpl_rx_data_ack *)skb->head;
+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));
+ req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |
+ V_RX_CREDITS(credits));
+ skb->priority = CPL_PRIORITY_ACK;
+ cxgb3_ofld_send(csk->cdev->lldev, skb);
+ return credits;
+}
+
+/*
+ * CPL connection tx data: host ->
+ *
+ * Send iscsi PDU via TX_DATA CPL message. Returns the number of
+ * credits sent.
+ * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
+ * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
+ */
+
+static unsigned int wrlen __read_mostly;
+static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
+
+static void init_wr_tab(unsigned int wr_len)
+{
+ int i;
+
+ if (skb_wrs[1]) /* already initialized */
+ return;
+ for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
+ int sgl_len = (3 * i) / 2 + (i & 1);
+
+ sgl_len += 3;
+ skb_wrs[i] = (sgl_len <= wr_len
+ ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
+ }
+ wrlen = wr_len * 8;
+}
+
+static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
+ int len, int req_completion)
+{
+ struct tx_data_wr *req;
+ struct l2t_entry *l2t = csk->l2t;
+
+ skb_reset_transport_header(skb);
+ req = __skb_push(skb, sizeof(*req));
+ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
+ (req_completion ? F_WR_COMPL : 0));
+ req->wr_lo = htonl(V_WR_TID(csk->tid));
+ /* len includes the length of any HW ULP additions */
+ req->len = htonl(len);
+ /* V_TX_ULP_SUBMODE sets both the mode and submode */
+ req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_tx_ulp_mode(skb)) |
+ V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
+ req->sndseq = htonl(csk->snd_nxt);
+ req->param = htonl(V_TX_PORT(l2t->smt_idx));
+
+ if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
+ req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
+ V_TX_CPU_IDX(csk->rss_qid));
+ /* sendbuffer is in units of 32KB. */
+ req->param |= htonl(V_TX_SNDBUF(csk->snd_win >> 15));
+ cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
+ }
+}
+
+/*
+ * push_tx_frames -- start transmit
+ *
+ * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
+ * connection's send queue and sends them on to T3. Must be called with the
+ * connection's lock held. Returns the amount of send buffer space that was
+ * freed as a result of sending queued data to T3.
+ */
+
+static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)
+{
+ kfree_skb(skb);
+}
+
+static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
+{
+ int total_size = 0;
+ struct sk_buff *skb;
+
+ if (unlikely(csk->state < CTP_ESTABLISHED ||
+ csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ return 0;
+ }
+
+ while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
+ int len = skb->len; /* length before skb_push */
+ int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
+ int wrs_needed = skb_wrs[frags];
+
+ if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
+ wrs_needed = 1;
+
+ WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
+
+ if (csk->wr_cred < wrs_needed) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
+ csk, skb->len, skb->data_len, frags,
+ wrs_needed, csk->wr_cred);
+ break;
+ }
+
+ __skb_unlink(skb, &csk->write_queue);
+ skb->priority = CPL_PRIORITY_DATA;
+ skb->csum = wrs_needed; /* remember this until the WR_ACK */
+ csk->wr_cred -= wrs_needed;
+ csk->wr_una_cred += wrs_needed;
+ cxgbi_sock_enqueue_wr(csk, skb);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
+ "left %u, unack %u.\n",
+ csk, skb->len, skb->data_len, frags, skb->csum,
+ csk->wr_cred, csk->wr_una_cred);
+
+ if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
+ if ((req_completion &&
+ csk->wr_una_cred == wrs_needed) ||
+ csk->wr_una_cred >= csk->wr_max_cred / 2) {
+ req_completion = 1;
+ csk->wr_una_cred = 0;
+ }
+ len += cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));
+ make_tx_data_wr(csk, skb, len, req_completion);
+ csk->snd_nxt += len;
+ cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
+ }
+ total_size += skb->truesize;
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p, tid 0x%x, send skb 0x%p.\n",
+ csk, csk->tid, skb);
+ set_arp_failure_handler(skb, arp_failure_skb_discard);
+ l2t_send(csk->cdev->lldev, skb, csk->l2t);
+ }
+ return total_size;
+}
+
+/*
+ * Process a CPL_ACT_ESTABLISH message: -> host
+ * Updates connection state from an active establish CPL message. Runs with
+ * the connection lock held.
+ */
+
+static inline void free_atid(struct cxgbi_sock *csk)
+{
+ if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
+ cxgb3_free_atid(csk->cdev->lldev, csk->atid);
+ cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
+ cxgbi_sock_put(csk);
+ }
+}
+
+static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
+{
+ struct cxgbi_sock *csk = ctx;
+ struct cpl_act_establish *req = cplhdr(skb);
+ unsigned int tid = GET_TID(req);
+ unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
+ u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
+ atid, atid, csk, csk->state, csk->flags, rcv_isn);
+
+ cxgbi_sock_get(csk);
+ cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
+ csk->tid = tid;
+ cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid);
+
+ free_atid(csk);
+
+ csk->rss_qid = G_QNUM(ntohs(skb->csum));
+
+ spin_lock_bh(&csk->lock);
+ if (csk->retry_timer.function) {
+ del_timer(&csk->retry_timer);
+ csk->retry_timer.function = NULL;
+ }
+
+ if (unlikely(csk->state != CTP_ACTIVE_OPEN))
+ pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
+ if (csk->rcv_win > (M_RCV_BUFSIZ << 10))
+ csk->rcv_wup -= csk->rcv_win - (M_RCV_BUFSIZ << 10);
+
+ cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
+
+ if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
+ /* upper layer has requested closing */
+ send_abort_req(csk);
+ else {
+ if (skb_queue_len(&csk->write_queue))
+ push_tx_frames(csk, 1);
+ cxgbi_conn_tx_open(csk);
+ }
+
+ spin_unlock_bh(&csk->lock);
+ __kfree_skb(skb);
+ return 0;
+}
+
+/*
+ * Process a CPL_ACT_OPEN_RPL message: -> host
+ * Handle active open failures.
+ */
+static int act_open_rpl_status_to_errno(int status)
+{
+ switch (status) {
+ case CPL_ERR_CONN_RESET:
+ return -ECONNREFUSED;
+ case CPL_ERR_ARP_MISS:
+ return -EHOSTUNREACH;
+ case CPL_ERR_CONN_TIMEDOUT:
+ return -ETIMEDOUT;
+ case CPL_ERR_TCAM_FULL:
+ return -ENOMEM;
+ case CPL_ERR_CONN_EXIST:
+ return -EADDRINUSE;
+ default:
+ return -EIO;
+ }
+}
+
+static void act_open_retry_timer(struct timer_list *t)
+{
+ struct cxgbi_sock *csk = from_timer(csk, t, retry_timer);
+ struct sk_buff *skb;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+ skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);
+ if (!skb)
+ cxgbi_sock_fail_act_open(csk, -ENOMEM);
+ else {
+ skb->sk = (struct sock *)csk;
+ set_arp_failure_handler(skb, act_open_arp_failure);
+ send_act_open_req(csk, skb, csk->l2t);
+ }
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+}
+
+static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
+{
+ struct cxgbi_sock *csk = ctx;
+ struct cpl_act_open_rpl *rpl = cplhdr(skb);
+
+ pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
+ csk, csk->state, csk->flags, csk->atid, rpl->status,
+ &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
+ &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
+
+ if (rpl->status != CPL_ERR_TCAM_FULL &&
+ rpl->status != CPL_ERR_CONN_EXIST &&
+ rpl->status != CPL_ERR_ARP_MISS)
+ cxgb3_queue_tid_release(tdev, GET_TID(rpl));
+
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+ if (rpl->status == CPL_ERR_CONN_EXIST &&
+ csk->retry_timer.function != act_open_retry_timer) {
+ csk->retry_timer.function = act_open_retry_timer;
+ mod_timer(&csk->retry_timer, jiffies + HZ / 2);
+ } else
+ cxgbi_sock_fail_act_open(csk,
+ act_open_rpl_status_to_errno(rpl->status));
+
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+ __kfree_skb(skb);
+ return 0;
+}
+
+/*
+ * Process PEER_CLOSE CPL messages: -> host
+ * Handle peer FIN.
+ */
+static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+{
+ struct cxgbi_sock *csk = ctx;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ cxgbi_sock_rcv_peer_close(csk);
+ __kfree_skb(skb);
+ return 0;
+}
+
+/*
+ * Process CLOSE_CONN_RPL CPL message: -> host
+ * Process a peer ACK to our FIN.
+ */
+static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
+ void *ctx)
+{
+ struct cxgbi_sock *csk = ctx;
+ struct cpl_close_con_rpl *rpl = cplhdr(skb);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
+ csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));
+
+ cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
+ __kfree_skb(skb);
+ return 0;
+}
+
+/*
+ * Process ABORT_REQ_RSS CPL message: -> host
+ * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
+ * request except that we need to reply to it.
+ */
+
+static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
+ int *need_rst)
+{
+ switch (abort_reason) {
+ case CPL_ERR_BAD_SYN:
+ case CPL_ERR_CONN_RESET:
+ return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;
+ case CPL_ERR_XMIT_TIMEDOUT:
+ case CPL_ERR_PERSIST_TIMEDOUT:
+ case CPL_ERR_FINWAIT2_TIMEDOUT:
+ case CPL_ERR_KEEPALIVE_TIMEDOUT:
+ return -ETIMEDOUT;
+ default:
+ return -EIO;
+ }
+}
+
+static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+{
+ const struct cpl_abort_req_rss *req = cplhdr(skb);
+ struct cxgbi_sock *csk = ctx;
+ int rst_status = CPL_ABORT_NO_RST;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
+ req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
+ goto done;
+ }
+
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+
+ if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {
+ cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
+ cxgbi_sock_set_state(csk, CTP_ABORTING);
+ goto out;
+ }
+
+ cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
+ send_abort_rpl(csk, rst_status);
+
+ if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
+ csk->err = abort_status_to_errno(csk, req->status, &rst_status);
+ cxgbi_sock_closed(csk);
+ }
+
+out:
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+done:
+ __kfree_skb(skb);
+ return 0;
+}
+
+/*
+ * Process ABORT_RPL_RSS CPL message: -> host
+ * Process abort replies. We only process these messages if we anticipate
+ * them as the coordination between SW and HW in this area is somewhat lacking
+ * and sometimes we get ABORT_RPLs after we are done with the connection that
+ * originated the ABORT_REQ.
+ */
+static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+{
+ struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
+ struct cxgbi_sock *csk = ctx;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
+ rpl->status, csk, csk ? csk->state : 0,
+ csk ? csk->flags : 0UL);
+ /*
+ * Ignore replies to post-close aborts indicating that the abort was
+ * requested too late. These connections are terminated when we get
+ * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
+ * arrives the TID is either no longer used or it has been recycled.
+ */
+ if (rpl->status == CPL_ERR_ABORT_FAILED)
+ goto rel_skb;
+ /*
+ * Sometimes we've already closed the connection, e.g., a post-close
+ * abort races with ABORT_REQ_RSS, the latter frees the connection
+ * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
+ * but FW turns the ABORT_REQ into a regular one and so we get
+ * ABORT_RPL_RSS with status 0 and no connection.
+ */
+ if (csk)
+ cxgbi_sock_rcv_abort_rpl(csk);
+rel_skb:
+ __kfree_skb(skb);
+ return 0;
+}
+
+/*
+ * Process RX_ISCSI_HDR CPL message: -> host
+ * Handle received PDUs, the payload could be DDP'ed. If not, the payload
+ * follow after the bhs.
+ */
+static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
+{
+ struct cxgbi_sock *csk = ctx;
+ struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
+ struct cpl_iscsi_hdr_norss data_cpl;
+ struct cpl_rx_data_ddp_norss ddp_cpl;
+ unsigned int hdr_len, data_len, status;
+ unsigned int len;
+ int err;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
+ csk, csk->state, csk->flags, csk->tid, skb, skb->len);
+
+ spin_lock_bh(&csk->lock);
+
+ if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, bad state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ if (csk->state != CTP_ABORTING)
+ goto abort_conn;
+ else
+ goto discard;
+ }
+
+ cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
+ cxgbi_skcb_flags(skb) = 0;
+
+ skb_reset_transport_header(skb);
+ __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
+
+ len = hdr_len = ntohs(hdr_cpl->len);
+ /* msg coalesce is off or not enough data received */
+ if (skb->len <= hdr_len) {
+ pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
+ csk->cdev->ports[csk->port_id]->name, csk->tid,
+ skb->len, hdr_len);
+ goto abort_conn;
+ }
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED);
+
+ err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
+ sizeof(ddp_cpl));
+ if (err < 0) {
+ pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
+ csk->cdev->ports[csk->port_id]->name, csk->tid,
+ skb->len, sizeof(ddp_cpl), err);
+ goto abort_conn;
+ }
+
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);
+ cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
+ cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
+ status = ntohl(ddp_cpl.ddp_status);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
+ csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);
+
+ if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);
+ if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);
+ if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);
+
+ if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
+ err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
+ if (err < 0) {
+ pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
+ csk->cdev->ports[csk->port_id]->name,
+ csk->tid, sizeof(data_cpl), skb->len, err);
+ goto abort_conn;
+ }
+ data_len = ntohs(data_cpl.len);
+ log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,
+ "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
+ skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);
+ len += sizeof(data_cpl) + data_len;
+ } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);
+
+ csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);
+ __pskb_trim(skb, len);
+ __skb_queue_tail(&csk->receive_queue, skb);
+ cxgbi_conn_pdu_ready(csk);
+
+ spin_unlock_bh(&csk->lock);
+ return 0;
+
+abort_conn:
+ send_abort_req(csk);
+discard:
+ spin_unlock_bh(&csk->lock);
+ __kfree_skb(skb);
+ return 0;
+}
+
+/*
+ * Process TX_DATA_ACK CPL messages: -> host
+ * Process an acknowledgment of WR completion. Advance snd_una and send the
+ * next batch of work requests from the write queue.
+ */
+static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
+{
+ struct cxgbi_sock *csk = ctx;
+ struct cpl_wr_ack *hdr = cplhdr(skb);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx,%u, cr %u.\n",
+ csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));
+
+ cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);
+ __kfree_skb(skb);
+ return 0;
+}
+
+/*
+ * for each connection, pre-allocate skbs needed for close/abort requests. So
+ * that we can service the request right away.
+ */
+static int alloc_cpls(struct cxgbi_sock *csk)
+{
+ csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0,
+ GFP_KERNEL);
+ if (!csk->cpl_close)
+ return -ENOMEM;
+ csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0,
+ GFP_KERNEL);
+ if (!csk->cpl_abort_req)
+ goto free_cpl_skbs;
+
+ csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0,
+ GFP_KERNEL);
+ if (!csk->cpl_abort_rpl)
+ goto free_cpl_skbs;
+
+ return 0;
+
+free_cpl_skbs:
+ cxgbi_sock_free_cpl_skbs(csk);
+ return -ENOMEM;
+}
+
+static void l2t_put(struct cxgbi_sock *csk)
+{
+ struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
+
+ if (csk->l2t) {
+ l2t_release(t3dev, csk->l2t);
+ csk->l2t = NULL;
+ cxgbi_sock_put(csk);
+ }
+}
+
+/*
+ * release_offload_resources - release offload resource
+ * Release resources held by an offload connection (TID, L2T entry, etc.)
+ */
+static void release_offload_resources(struct cxgbi_sock *csk)
+{
+ struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ csk->rss_qid = 0;
+ cxgbi_sock_free_cpl_skbs(csk);
+
+ if (csk->wr_cred != csk->wr_max_cred) {
+ cxgbi_sock_purge_wr_queue(csk);
+ cxgbi_sock_reset_wr_list(csk);
+ }
+ l2t_put(csk);
+ if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
+ free_atid(csk);
+ else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
+ cxgb3_remove_tid(t3dev, (void *)csk, csk->tid);
+ cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
+ cxgbi_sock_put(csk);
+ }
+ csk->dst = NULL;
+ csk->cdev = NULL;
+}
+
+static void update_address(struct cxgbi_hba *chba)
+{
+ if (chba->ipv4addr) {
+ if (chba->vdev &&
+ chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) {
+ cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr);
+ cxgb3i_set_private_ipv4addr(chba->ndev, 0);
+ pr_info("%s set %pI4.\n",
+ chba->vdev->name, &chba->ipv4addr);
+ } else if (chba->ipv4addr !=
+ cxgb3i_get_private_ipv4addr(chba->ndev)) {
+ cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr);
+ pr_info("%s set %pI4.\n",
+ chba->ndev->name, &chba->ipv4addr);
+ }
+ } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) {
+ if (chba->vdev)
+ cxgb3i_set_private_ipv4addr(chba->vdev, 0);
+ cxgb3i_set_private_ipv4addr(chba->ndev, 0);
+ }
+}
+
+static int init_act_open(struct cxgbi_sock *csk)
+{
+ struct dst_entry *dst = csk->dst;
+ struct cxgbi_device *cdev = csk->cdev;
+ struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
+ struct net_device *ndev = cdev->ports[csk->port_id];
+ struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
+ struct sk_buff *skb = NULL;
+ int ret;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);
+
+ update_address(chba);
+ if (chba->ipv4addr)
+ csk->saddr.sin_addr.s_addr = chba->ipv4addr;
+
+ csk->rss_qid = 0;
+ csk->l2t = t3_l2t_get(t3dev, dst, ndev,
+ &csk->daddr.sin_addr.s_addr);
+ if (!csk->l2t) {
+ pr_err("NO l2t available.\n");
+ return -EINVAL;
+ }
+ cxgbi_sock_get(csk);
+
+ csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
+ if (csk->atid < 0) {
+ pr_err("NO atid available.\n");
+ ret = -EINVAL;
+ goto put_sock;
+ }
+ cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
+ cxgbi_sock_get(csk);
+
+ skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
+ if (!skb) {
+ ret = -ENOMEM;
+ goto free_atid;
+ }
+ skb->sk = (struct sock *)csk;
+ set_arp_failure_handler(skb, act_open_arp_failure);
+ csk->snd_win = cxgb3i_snd_win;
+ csk->rcv_win = cxgb3i_rcv_win;
+
+ csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;
+ csk->wr_una_cred = 0;
+ csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));
+ cxgbi_sock_reset_wr_list(csk);
+ csk->err = 0;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
+ csk, csk->state, csk->flags,
+ &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
+ &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
+
+ cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
+ send_act_open_req(csk, skb, csk->l2t);
+ return 0;
+
+free_atid:
+ cxgb3_free_atid(t3dev, csk->atid);
+put_sock:
+ cxgbi_sock_put(csk);
+ l2t_release(t3dev, csk->l2t);
+ csk->l2t = NULL;
+
+ return ret;
+}
+
+cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
+ [CPL_ACT_ESTABLISH] = do_act_establish,
+ [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
+ [CPL_PEER_CLOSE] = do_peer_close,
+ [CPL_ABORT_REQ_RSS] = do_abort_req,
+ [CPL_ABORT_RPL_RSS] = do_abort_rpl,
+ [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
+ [CPL_TX_DMA_ACK] = do_wr_ack,
+ [CPL_ISCSI_HDR] = do_iscsi_hdr,
+};
+
+/**
+ * cxgb3i_ofld_init - allocate and initialize resources for each adapter found
+ * @cdev: cxgbi adapter
+ */
+static int cxgb3i_ofld_init(struct cxgbi_device *cdev)
+{
+ struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
+ struct adap_ports port;
+ struct ofld_page_info rx_page_info;
+ unsigned int wr_len;
+ int rc;
+
+ if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||
+ t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||
+ t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
+ pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);
+ return -EINVAL;
+ }
+
+ if (cxgb3i_max_connect > CXGBI_MAX_CONN)
+ cxgb3i_max_connect = CXGBI_MAX_CONN;
+
+ rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base,
+ cxgb3i_max_connect);
+ if (rc < 0)
+ return rc;
+
+ init_wr_tab(wr_len);
+ cdev->csk_release_offload_resources = release_offload_resources;
+ cdev->csk_push_tx_frames = push_tx_frames;
+ cdev->csk_send_abort_req = send_abort_req;
+ cdev->csk_send_close_req = send_close_req;
+ cdev->csk_send_rx_credits = send_rx_credits;
+ cdev->csk_alloc_cpls = alloc_cpls;
+ cdev->csk_init_act_open = init_act_open;
+
+ pr_info("cdev 0x%p, offload up, added.\n", cdev);
+ return 0;
+}
+
+/*
+ * functions to program the pagepod in h/w
+ */
+static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
+{
+ struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
+
+ memset(req, 0, sizeof(*req));
+
+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
+ req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
+ V_ULPTX_CMD(ULP_MEM_WRITE));
+ req->len = htonl(V_ULP_MEMIO_DATA_LEN(IPPOD_SIZE >> 5) |
+ V_ULPTX_NFLITS((IPPOD_SIZE >> 3) + 1));
+}
+
+static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev)
+{
+ return ((struct t3cdev *)cdev->lldev)->ulp_iscsi;
+}
+
+static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,
+ struct cxgbi_task_tag_info *ttinfo)
+{
+ unsigned int idx = ttinfo->idx;
+ unsigned int npods = ttinfo->npods;
+ struct scatterlist *sg = ttinfo->sgl;
+ struct cxgbi_pagepod *ppod;
+ struct ulp_mem_io *req;
+ unsigned int sg_off;
+ unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
+ int i;
+
+ for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {
+ struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
+ IPPOD_SIZE, 0, GFP_ATOMIC);
+
+ if (!skb)
+ return -ENOMEM;
+ ulp_mem_io_set_hdr(skb, pm_addr);
+ req = (struct ulp_mem_io *)skb->head;
+ ppod = (struct cxgbi_pagepod *)(req + 1);
+ sg_off = i * PPOD_PAGES_MAX;
+ cxgbi_ddp_set_one_ppod(ppod, ttinfo, &sg,
+ &sg_off);
+ skb->priority = CPL_PRIORITY_CONTROL;
+ cxgb3_ofld_send(ppm->lldev, skb);
+ }
+ return 0;
+}
+
+static void ddp_clear_map(struct cxgbi_device *cdev, struct cxgbi_ppm *ppm,
+ struct cxgbi_task_tag_info *ttinfo)
+{
+ unsigned int idx = ttinfo->idx;
+ unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
+ unsigned int npods = ttinfo->npods;
+ int i;
+
+ log_debug(1 << CXGBI_DBG_DDP,
+ "cdev 0x%p, clear idx %u, npods %u.\n",
+ cdev, idx, npods);
+
+ for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {
+ struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
+ IPPOD_SIZE, 0, GFP_ATOMIC);
+
+ if (!skb) {
+ pr_err("cdev 0x%p, clear ddp, %u,%d/%u, skb OOM.\n",
+ cdev, idx, i, npods);
+ continue;
+ }
+ ulp_mem_io_set_hdr(skb, pm_addr);
+ skb->priority = CPL_PRIORITY_CONTROL;
+ cxgb3_ofld_send(ppm->lldev, skb);
+ }
+}
+
+static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,
+ unsigned int tid, int pg_idx)
+{
+ struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
+ GFP_KERNEL);
+ struct cpl_set_tcb_field *req;
+ u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;
+
+ log_debug(1 << CXGBI_DBG_DDP,
+ "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);
+ if (!skb)
+ return -ENOMEM;
+
+ /* set up ulp submode and page size */
+ req = (struct cpl_set_tcb_field *)skb->head;
+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
+ req->reply = V_NO_REPLY(1);
+ req->cpu_idx = 0;
+ req->word = htons(31);
+ req->mask = cpu_to_be64(0xF0000000);
+ req->val = cpu_to_be64(val << 28);
+ skb->priority = CPL_PRIORITY_CONTROL;
+
+ cxgb3_ofld_send(csk->cdev->lldev, skb);
+ return 0;
+}
+
+/**
+ * cxgb3i_setup_conn_digest - setup conn. digest setting
+ * @csk: cxgb tcp socket
+ * @tid: connection id
+ * @hcrc: header digest enabled
+ * @dcrc: data digest enabled
+ * set up the iscsi digest settings for a connection identified by tid
+ */
+static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
+ int hcrc, int dcrc)
+{
+ struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
+ GFP_KERNEL);
+ struct cpl_set_tcb_field *req;
+ u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
+
+ log_debug(1 << CXGBI_DBG_DDP,
+ "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);
+ if (!skb)
+ return -ENOMEM;
+
+ /* set up ulp submode and page size */
+ req = (struct cpl_set_tcb_field *)skb->head;
+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
+ req->reply = V_NO_REPLY(1);
+ req->cpu_idx = 0;
+ req->word = htons(31);
+ req->mask = cpu_to_be64(0x0F000000);
+ req->val = cpu_to_be64(val << 24);
+ skb->priority = CPL_PRIORITY_CONTROL;
+
+ cxgb3_ofld_send(csk->cdev->lldev, skb);
+ return 0;
+}
+
+/**
+ * cxgb3i_ddp_init - initialize the cxgb3 adapter's ddp resource
+ * @cdev: cxgb3i adapter
+ * initialize the ddp pagepod manager for a given adapter
+ */
+static int cxgb3i_ddp_init(struct cxgbi_device *cdev)
+{
+ struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
+ struct net_device *ndev = cdev->ports[0];
+ struct cxgbi_tag_format tformat;
+ unsigned int ppmax, tagmask = 0;
+ struct ulp_iscsi_info uinfo;
+ int i, err;
+
+ err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
+ if (err < 0) {
+ pr_err("%s, failed to get iscsi param %d.\n",
+ ndev->name, err);
+ return err;
+ }
+ if (uinfo.llimit >= uinfo.ulimit) {
+ pr_warn("T3 %s, iscsi NOT enabled %u ~ %u!\n",
+ ndev->name, uinfo.llimit, uinfo.ulimit);
+ return -EACCES;
+ }
+
+ ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
+ tagmask = cxgbi_tagmask_set(ppmax);
+
+ pr_info("T3 %s: 0x%x~0x%x, 0x%x, tagmask 0x%x -> 0x%x.\n",
+ ndev->name, uinfo.llimit, uinfo.ulimit, ppmax, uinfo.tagmask,
+ tagmask);
+
+ memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
+ for (i = 0; i < 4; i++)
+ tformat.pgsz_order[i] = uinfo.pgsz_factor[i];
+ cxgbi_tagmask_check(tagmask, &tformat);
+
+ err = cxgbi_ddp_ppm_setup(&tdev->ulp_iscsi, cdev, &tformat,
+ (uinfo.ulimit - uinfo.llimit + 1),
+ uinfo.llimit, uinfo.llimit, 0, 0, 0);
+ if (err)
+ return err;
+
+ if (!(cdev->flags & CXGBI_FLAG_DDP_OFF)) {
+ uinfo.tagmask = tagmask;
+ uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
+
+ err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
+ if (err < 0) {
+ pr_err("T3 %s fail to set iscsi param %d.\n",
+ ndev->name, err);
+ cdev->flags |= CXGBI_FLAG_DDP_OFF;
+ }
+ err = 0;
+ }
+
+ cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
+ cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
+ cdev->csk_ddp_set_map = ddp_set_map;
+ cdev->csk_ddp_clear_map = ddp_clear_map;
+ cdev->cdev2ppm = cdev2ppm;
+ cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
+ uinfo.max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
+ cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
+ uinfo.max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
+
+ return 0;
+}
+
+static void cxgb3i_dev_close(struct t3cdev *t3dev)
+{
+ struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
+
+ if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {
+ pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);
+ return;
+ }
+
+ cxgbi_device_unregister(cdev);
+}
+
+/**
+ * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
+ * @t3dev: t3cdev adapter
+ */
+static void cxgb3i_dev_open(struct t3cdev *t3dev)
+{
+ struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
+ struct adapter *adapter = tdev2adap(t3dev);
+ int i, err;
+
+ if (cdev) {
+ pr_info("0x%p, updating.\n", cdev);
+ return;
+ }
+
+ cdev = cxgbi_device_register(0, adapter->params.nports);
+ if (!cdev) {
+ pr_warn("device 0x%p register failed.\n", t3dev);
+ return;
+ }
+
+ cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;
+ cdev->lldev = t3dev;
+ cdev->pdev = adapter->pdev;
+ cdev->ports = adapter->port;
+ cdev->nports = adapter->params.nports;
+ cdev->mtus = adapter->params.mtus;
+ cdev->nmtus = NMTUS;
+ cdev->rx_credit_thres = cxgb3i_rx_credit_thres;
+ cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;
+ cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);
+ cdev->itp = &cxgb3i_iscsi_transport;
+
+ err = cxgb3i_ddp_init(cdev);
+ if (err) {
+ pr_info("0x%p ddp init failed %d\n", cdev, err);
+ goto err_out;
+ }
+
+ err = cxgb3i_ofld_init(cdev);
+ if (err) {
+ pr_info("0x%p offload init failed\n", cdev);
+ goto err_out;
+ }
+
+ err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,
+ &cxgb3i_host_template, cxgb3i_stt);
+ if (err)
+ goto err_out;
+
+ for (i = 0; i < cdev->nports; i++)
+ cdev->hbas[i]->ipv4addr =
+ cxgb3i_get_private_ipv4addr(cdev->ports[i]);
+
+ pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
+ cdev, cdev ? cdev->flags : 0, t3dev, err);
+ return;
+
+err_out:
+ cxgbi_device_unregister(cdev);
+}
+
+static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)
+{
+ struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
+
+ log_debug(1 << CXGBI_DBG_TOE,
+ "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
+ t3dev, cdev, event, port);
+ if (!cdev)
+ return;
+
+ switch (event) {
+ case OFFLOAD_STATUS_DOWN:
+ cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;
+ break;
+ case OFFLOAD_STATUS_UP:
+ cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;
+ break;
+ }
+}
+
+/**
+ * cxgb3i_init_module - module init entry point
+ *
+ * initialize any driver wide global data structures and register itself
+ * with the cxgb3 module
+ */
+static int __init cxgb3i_init_module(void)
+{
+ int rc;
+
+ printk(KERN_INFO "%s", version);
+
+ rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);
+ if (rc < 0)
+ return rc;
+
+ cxgb3_register_client(&t3_client);
+ return 0;
+}
+
+/**
+ * cxgb3i_exit_module - module cleanup/exit entry point
+ *
+ * go through the driver hba list and for each hba, release any resource held.
+ * and unregisters iscsi transport and the cxgb3 module
+ */
+static void __exit cxgb3i_exit_module(void)
+{
+ cxgb3_unregister_client(&t3_client);
+ cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);
+ cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);
+}
+
+module_init(cxgb3i_init_module);
+module_exit(cxgb3i_exit_module);
diff --git a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.h b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.h
new file mode 100644
index 000000000..b0430c935
--- /dev/null
+++ b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.h
@@ -0,0 +1,62 @@
+/*
+ * cxgb3i.h: Chelsio S3xx iSCSI driver.
+ *
+ * Copyright (c) 2008-2015 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * Written by: Karen Xie (kxie@chelsio.com)
+ */
+
+#ifndef __CXGB3I_H__
+#define __CXGB3I_H__
+
+#define CXGB3I_SCSI_HOST_QDEPTH 1024
+#define CXGB3I_MAX_LUN 512
+#define ISCSI_PDU_NONPAYLOAD_MAX \
+ (sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE + 2*ISCSI_DIGEST_SIZE)
+
+/*for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
+#define CXGB3I_TX_HEADER_LEN \
+ (sizeof(struct tx_data_wr) + sizeof(struct sge_opaque_hdr))
+
+extern cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS];
+
+static inline unsigned int cxgb3i_get_private_ipv4addr(struct net_device *ndev)
+{
+ return ((struct port_info *)(netdev_priv(ndev)))->iscsi_ipv4addr;
+}
+
+static inline void cxgb3i_set_private_ipv4addr(struct net_device *ndev,
+ unsigned int addr)
+{
+ struct port_info *pi = (struct port_info *)netdev_priv(ndev);
+
+ pi->iscsic.flags = addr ? 1 : 0;
+ pi->iscsi_ipv4addr = addr;
+ if (addr)
+ memcpy(pi->iscsic.mac_addr, ndev->dev_addr, ETH_ALEN);
+}
+
+struct cpl_iscsi_hdr_norss {
+ union opcode_tid ot;
+ u16 pdu_len_ddp;
+ u16 len;
+ u32 seq;
+ u16 urg;
+ u8 rsvd;
+ u8 status;
+};
+
+struct cpl_rx_data_ddp_norss {
+ union opcode_tid ot;
+ u16 urg;
+ u16 len;
+ u32 seq;
+ u32 nxt_seq;
+ u32 ulp_crc;
+ u32 ddp_status;
+};
+#endif
diff --git a/drivers/scsi/cxgbi/cxgb4i/Kbuild b/drivers/scsi/cxgbi/cxgb4i/Kbuild
new file mode 100644
index 000000000..fd3e0c964
--- /dev/null
+++ b/drivers/scsi/cxgbi/cxgb4i/Kbuild
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+ccflags-y += -I$(srctree)/drivers/net/ethernet/chelsio/cxgb4
+ccflags-y += -I$(srctree)/drivers/net/ethernet/chelsio/libcxgb
+
+obj-$(CONFIG_SCSI_CXGB4_ISCSI) += cxgb4i.o
diff --git a/drivers/scsi/cxgbi/cxgb4i/Kconfig b/drivers/scsi/cxgbi/cxgb4i/Kconfig
new file mode 100644
index 000000000..8b0deece9
--- /dev/null
+++ b/drivers/scsi/cxgbi/cxgb4i/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config SCSI_CXGB4_ISCSI
+ tristate "Chelsio T4 iSCSI support"
+ depends on PCI && INET && (IPV6 || IPV6=n)
+ depends on THERMAL || !THERMAL
+ depends on ETHERNET
+ depends on TLS || TLS=n
+ select NET_VENDOR_CHELSIO
+ select CHELSIO_T4
+ select CHELSIO_LIB
+ select SCSI_ISCSI_ATTRS
+ help
+ This driver supports iSCSI offload for the Chelsio T4 devices.
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
new file mode 100644
index 000000000..efb3e2b33
--- /dev/null
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
@@ -0,0 +1,2472 @@
+/*
+ * cxgb4i.c: Chelsio T4 iSCSI driver.
+ *
+ * Copyright (c) 2010-2015 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * Written by: Karen Xie (kxie@chelsio.com)
+ * Rakesh Ranjan (rranjan@chelsio.com)
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <scsi/scsi_host.h>
+#include <net/tcp.h>
+#include <net/dst.h>
+#include <linux/netdevice.h>
+#include <net/addrconf.h>
+
+#include "t4_regs.h"
+#include "t4_msg.h"
+#include "cxgb4.h"
+#include "cxgb4_uld.h"
+#include "t4fw_api.h"
+#include "l2t.h"
+#include "cxgb4i.h"
+#include "clip_tbl.h"
+
+static unsigned int dbg_level;
+
+#include "../libcxgbi.h"
+
+#ifdef CONFIG_CHELSIO_T4_DCB
+#include <net/dcbevent.h>
+#include "cxgb4_dcb.h"
+#endif
+
+#define DRV_MODULE_NAME "cxgb4i"
+#define DRV_MODULE_DESC "Chelsio T4-T6 iSCSI Driver"
+#define DRV_MODULE_VERSION "0.9.5-ko"
+#define DRV_MODULE_RELDATE "Apr. 2015"
+
+static char version[] =
+ DRV_MODULE_DESC " " DRV_MODULE_NAME
+ " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
+
+MODULE_AUTHOR("Chelsio Communications, Inc.");
+MODULE_DESCRIPTION(DRV_MODULE_DESC);
+MODULE_VERSION(DRV_MODULE_VERSION);
+MODULE_LICENSE("GPL");
+
+module_param(dbg_level, uint, 0644);
+MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)");
+
+#define CXGB4I_DEFAULT_10G_RCV_WIN (256 * 1024)
+static int cxgb4i_rcv_win = -1;
+module_param(cxgb4i_rcv_win, int, 0644);
+MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP receive window in bytes");
+
+#define CXGB4I_DEFAULT_10G_SND_WIN (128 * 1024)
+static int cxgb4i_snd_win = -1;
+module_param(cxgb4i_snd_win, int, 0644);
+MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes");
+
+static int cxgb4i_rx_credit_thres = 10 * 1024;
+module_param(cxgb4i_rx_credit_thres, int, 0644);
+MODULE_PARM_DESC(cxgb4i_rx_credit_thres,
+ "RX credits return threshold in bytes (default=10KB)");
+
+static unsigned int cxgb4i_max_connect = (8 * 1024);
+module_param(cxgb4i_max_connect, uint, 0644);
+MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections");
+
+static unsigned short cxgb4i_sport_base = 20000;
+module_param(cxgb4i_sport_base, ushort, 0644);
+MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)");
+
+typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *);
+
+static void *t4_uld_add(const struct cxgb4_lld_info *);
+static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *);
+static int t4_uld_state_change(void *, enum cxgb4_state state);
+static inline int send_tx_flowc_wr(struct cxgbi_sock *);
+
+static const struct cxgb4_uld_info cxgb4i_uld_info = {
+ .name = DRV_MODULE_NAME,
+ .nrxq = MAX_ULD_QSETS,
+ .ntxq = MAX_ULD_QSETS,
+ .rxq_size = 1024,
+ .lro = false,
+ .add = t4_uld_add,
+ .rx_handler = t4_uld_rx_handler,
+ .state_change = t4_uld_state_change,
+};
+
+static struct scsi_host_template cxgb4i_host_template = {
+ .module = THIS_MODULE,
+ .name = DRV_MODULE_NAME,
+ .proc_name = DRV_MODULE_NAME,
+ .can_queue = CXGB4I_SCSI_HOST_QDEPTH,
+ .queuecommand = iscsi_queuecommand,
+ .change_queue_depth = scsi_change_queue_depth,
+ .sg_tablesize = SG_ALL,
+ .max_sectors = 0xFFFF,
+ .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
+ .eh_timed_out = iscsi_eh_cmd_timed_out,
+ .eh_abort_handler = iscsi_eh_abort,
+ .eh_device_reset_handler = iscsi_eh_device_reset,
+ .eh_target_reset_handler = iscsi_eh_recover_target,
+ .target_alloc = iscsi_target_alloc,
+ .dma_boundary = PAGE_SIZE - 1,
+ .this_id = -1,
+ .track_queue_depth = 1,
+};
+
+static struct iscsi_transport cxgb4i_iscsi_transport = {
+ .owner = THIS_MODULE,
+ .name = DRV_MODULE_NAME,
+ .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST |
+ CAP_DATADGST | CAP_DIGEST_OFFLOAD |
+ CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
+ .attr_is_visible = cxgbi_attr_is_visible,
+ .get_host_param = cxgbi_get_host_param,
+ .set_host_param = cxgbi_set_host_param,
+ /* session management */
+ .create_session = cxgbi_create_session,
+ .destroy_session = cxgbi_destroy_session,
+ .get_session_param = iscsi_session_get_param,
+ /* connection management */
+ .create_conn = cxgbi_create_conn,
+ .bind_conn = cxgbi_bind_conn,
+ .unbind_conn = iscsi_conn_unbind,
+ .destroy_conn = iscsi_tcp_conn_teardown,
+ .start_conn = iscsi_conn_start,
+ .stop_conn = iscsi_conn_stop,
+ .get_conn_param = iscsi_conn_get_param,
+ .set_param = cxgbi_set_conn_param,
+ .get_stats = cxgbi_get_conn_stats,
+ /* pdu xmit req from user space */
+ .send_pdu = iscsi_conn_send_pdu,
+ /* task */
+ .init_task = iscsi_tcp_task_init,
+ .xmit_task = iscsi_tcp_task_xmit,
+ .cleanup_task = cxgbi_cleanup_task,
+ /* pdu */
+ .alloc_pdu = cxgbi_conn_alloc_pdu,
+ .init_pdu = cxgbi_conn_init_pdu,
+ .xmit_pdu = cxgbi_conn_xmit_pdu,
+ .parse_pdu_itt = cxgbi_parse_pdu_itt,
+ /* TCP connect/disconnect */
+ .get_ep_param = cxgbi_get_ep_param,
+ .ep_connect = cxgbi_ep_connect,
+ .ep_poll = cxgbi_ep_poll,
+ .ep_disconnect = cxgbi_ep_disconnect,
+ /* Error recovery timeout call */
+ .session_recovery_timedout = iscsi_session_recovery_timedout,
+};
+
+#ifdef CONFIG_CHELSIO_T4_DCB
+static int
+cxgb4_dcb_change_notify(struct notifier_block *, unsigned long, void *);
+
+static struct notifier_block cxgb4_dcb_change = {
+ .notifier_call = cxgb4_dcb_change_notify,
+};
+#endif
+
+static struct scsi_transport_template *cxgb4i_stt;
+
+/*
+ * CPL (Chelsio Protocol Language) defines a message passing interface between
+ * the host driver and Chelsio asic.
+ * The section below implments CPLs that related to iscsi tcp connection
+ * open/close/abort and data send/receive.
+ */
+
+#define RCV_BUFSIZ_MASK 0x3FFU
+#define MAX_IMM_TX_PKT_LEN 256
+
+static int push_tx_frames(struct cxgbi_sock *, int);
+
+/*
+ * is_ofld_imm - check whether a packet can be sent as immediate data
+ * @skb: the packet
+ *
+ * Returns true if a packet can be sent as an offload WR with immediate
+ * data. We currently use the same limit as for Ethernet packets.
+ */
+static inline bool is_ofld_imm(const struct sk_buff *skb)
+{
+ int len = skb->len;
+
+ if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))
+ len += sizeof(struct fw_ofld_tx_data_wr);
+
+ if (likely(cxgbi_skcb_test_flag((struct sk_buff *)skb, SKCBF_TX_ISO)))
+ len += sizeof(struct cpl_tx_data_iso);
+
+ return (len <= MAX_IMM_OFLD_TX_DATA_WR_LEN);
+}
+
+static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
+ struct l2t_entry *e)
+{
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
+ int wscale = cxgbi_sock_compute_wscale(csk->mss_idx);
+ unsigned long long opt0;
+ unsigned int opt2;
+ unsigned int qid_atid = ((unsigned int)csk->atid) |
+ (((unsigned int)csk->rss_qid) << 14);
+
+ opt0 = KEEP_ALIVE_F |
+ WND_SCALE_V(wscale) |
+ MSS_IDX_V(csk->mss_idx) |
+ L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) |
+ TX_CHAN_V(csk->tx_chan) |
+ SMAC_SEL_V(csk->smac_idx) |
+ ULP_MODE_V(ULP_MODE_ISCSI) |
+ RCV_BUFSIZ_V(csk->rcv_win >> 10);
+
+ opt2 = RX_CHANNEL_V(0) |
+ RSS_QUEUE_VALID_F |
+ RSS_QUEUE_V(csk->rss_qid);
+
+ if (is_t4(lldi->adapter_type)) {
+ struct cpl_act_open_req *req =
+ (struct cpl_act_open_req *)skb->head;
+
+ INIT_TP_WR(req, 0);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
+ qid_atid));
+ req->local_port = csk->saddr.sin_port;
+ req->peer_port = csk->daddr.sin_port;
+ req->local_ip = csk->saddr.sin_addr.s_addr;
+ req->peer_ip = csk->daddr.sin_addr.s_addr;
+ req->opt0 = cpu_to_be64(opt0);
+ req->params = cpu_to_be32(cxgb4_select_ntuple(
+ csk->cdev->ports[csk->port_id],
+ csk->l2t));
+ opt2 |= RX_FC_VALID_F;
+ req->opt2 = cpu_to_be32(opt2);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
+ csk, &req->local_ip, ntohs(req->local_port),
+ &req->peer_ip, ntohs(req->peer_port),
+ csk->atid, csk->rss_qid);
+ } else if (is_t5(lldi->adapter_type)) {
+ struct cpl_t5_act_open_req *req =
+ (struct cpl_t5_act_open_req *)skb->head;
+ u32 isn = (prandom_u32() & ~7UL) - 1;
+
+ INIT_TP_WR(req, 0);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
+ qid_atid));
+ req->local_port = csk->saddr.sin_port;
+ req->peer_port = csk->daddr.sin_port;
+ req->local_ip = csk->saddr.sin_addr.s_addr;
+ req->peer_ip = csk->daddr.sin_addr.s_addr;
+ req->opt0 = cpu_to_be64(opt0);
+ req->params = cpu_to_be64(FILTER_TUPLE_V(
+ cxgb4_select_ntuple(
+ csk->cdev->ports[csk->port_id],
+ csk->l2t)));
+ req->rsvd = cpu_to_be32(isn);
+ opt2 |= T5_ISS_VALID;
+ opt2 |= T5_OPT_2_VALID_F;
+
+ req->opt2 = cpu_to_be32(opt2);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
+ csk, &req->local_ip, ntohs(req->local_port),
+ &req->peer_ip, ntohs(req->peer_port),
+ csk->atid, csk->rss_qid);
+ } else {
+ struct cpl_t6_act_open_req *req =
+ (struct cpl_t6_act_open_req *)skb->head;
+ u32 isn = (prandom_u32() & ~7UL) - 1;
+
+ INIT_TP_WR(req, 0);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
+ qid_atid));
+ req->local_port = csk->saddr.sin_port;
+ req->peer_port = csk->daddr.sin_port;
+ req->local_ip = csk->saddr.sin_addr.s_addr;
+ req->peer_ip = csk->daddr.sin_addr.s_addr;
+ req->opt0 = cpu_to_be64(opt0);
+ req->params = cpu_to_be64(FILTER_TUPLE_V(
+ cxgb4_select_ntuple(
+ csk->cdev->ports[csk->port_id],
+ csk->l2t)));
+ req->rsvd = cpu_to_be32(isn);
+
+ opt2 |= T5_ISS_VALID;
+ opt2 |= RX_FC_DISABLE_F;
+ opt2 |= T5_OPT_2_VALID_F;
+
+ req->opt2 = cpu_to_be32(opt2);
+ req->rsvd2 = cpu_to_be32(0);
+ req->opt3 = cpu_to_be32(0);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk t6 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
+ csk, &req->local_ip, ntohs(req->local_port),
+ &req->peer_ip, ntohs(req->peer_port),
+ csk->atid, csk->rss_qid);
+ }
+
+ set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
+
+ pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n",
+ (&csk->saddr), (&csk->daddr),
+ CHELSIO_CHIP_VERSION(lldi->adapter_type), csk,
+ csk->state, csk->flags, csk->atid, csk->rss_qid);
+
+ cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static void send_act_open_req6(struct cxgbi_sock *csk, struct sk_buff *skb,
+ struct l2t_entry *e)
+{
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
+ int wscale = cxgbi_sock_compute_wscale(csk->mss_idx);
+ unsigned long long opt0;
+ unsigned int opt2;
+ unsigned int qid_atid = ((unsigned int)csk->atid) |
+ (((unsigned int)csk->rss_qid) << 14);
+
+ opt0 = KEEP_ALIVE_F |
+ WND_SCALE_V(wscale) |
+ MSS_IDX_V(csk->mss_idx) |
+ L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) |
+ TX_CHAN_V(csk->tx_chan) |
+ SMAC_SEL_V(csk->smac_idx) |
+ ULP_MODE_V(ULP_MODE_ISCSI) |
+ RCV_BUFSIZ_V(csk->rcv_win >> 10);
+
+ opt2 = RX_CHANNEL_V(0) |
+ RSS_QUEUE_VALID_F |
+ RSS_QUEUE_V(csk->rss_qid);
+
+ if (is_t4(lldi->adapter_type)) {
+ struct cpl_act_open_req6 *req =
+ (struct cpl_act_open_req6 *)skb->head;
+
+ INIT_TP_WR(req, 0);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
+ qid_atid));
+ req->local_port = csk->saddr6.sin6_port;
+ req->peer_port = csk->daddr6.sin6_port;
+
+ req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr);
+ req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr +
+ 8);
+ req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr);
+ req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr +
+ 8);
+
+ req->opt0 = cpu_to_be64(opt0);
+
+ opt2 |= RX_FC_VALID_F;
+ req->opt2 = cpu_to_be32(opt2);
+
+ req->params = cpu_to_be32(cxgb4_select_ntuple(
+ csk->cdev->ports[csk->port_id],
+ csk->l2t));
+ } else if (is_t5(lldi->adapter_type)) {
+ struct cpl_t5_act_open_req6 *req =
+ (struct cpl_t5_act_open_req6 *)skb->head;
+
+ INIT_TP_WR(req, 0);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
+ qid_atid));
+ req->local_port = csk->saddr6.sin6_port;
+ req->peer_port = csk->daddr6.sin6_port;
+ req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr);
+ req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr +
+ 8);
+ req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr);
+ req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr +
+ 8);
+ req->opt0 = cpu_to_be64(opt0);
+
+ opt2 |= T5_OPT_2_VALID_F;
+ req->opt2 = cpu_to_be32(opt2);
+
+ req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple(
+ csk->cdev->ports[csk->port_id],
+ csk->l2t)));
+ } else {
+ struct cpl_t6_act_open_req6 *req =
+ (struct cpl_t6_act_open_req6 *)skb->head;
+
+ INIT_TP_WR(req, 0);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
+ qid_atid));
+ req->local_port = csk->saddr6.sin6_port;
+ req->peer_port = csk->daddr6.sin6_port;
+ req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr);
+ req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr +
+ 8);
+ req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr);
+ req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr +
+ 8);
+ req->opt0 = cpu_to_be64(opt0);
+
+ opt2 |= RX_FC_DISABLE_F;
+ opt2 |= T5_OPT_2_VALID_F;
+
+ req->opt2 = cpu_to_be32(opt2);
+
+ req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple(
+ csk->cdev->ports[csk->port_id],
+ csk->l2t)));
+
+ req->rsvd2 = cpu_to_be32(0);
+ req->opt3 = cpu_to_be32(0);
+ }
+
+ set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
+
+ pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n",
+ CHELSIO_CHIP_VERSION(lldi->adapter_type), csk, csk->state,
+ csk->flags, csk->atid,
+ &csk->saddr6.sin6_addr, ntohs(csk->saddr.sin_port),
+ &csk->daddr6.sin6_addr, ntohs(csk->daddr.sin_port),
+ csk->rss_qid);
+
+ cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
+}
+#endif
+
+static void send_close_req(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb = csk->cpl_close;
+ struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
+ unsigned int tid = csk->tid;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx, tid %u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ csk->cpl_close = NULL;
+ set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
+ INIT_TP_WR(req, tid);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
+ req->rsvd = 0;
+
+ cxgbi_sock_skb_entail(csk, skb);
+ if (csk->state >= CTP_ESTABLISHED)
+ push_tx_frames(csk, 1);
+}
+
+static void abort_arp_failure(void *handle, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk = (struct cxgbi_sock *)handle;
+ struct cpl_abort_req *req;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx, tid %u, abort.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ req = (struct cpl_abort_req *)skb->data;
+ req->cmd = CPL_ABORT_NO_RST;
+ cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
+}
+
+static void send_abort_req(struct cxgbi_sock *csk)
+{
+ struct cpl_abort_req *req;
+ struct sk_buff *skb = csk->cpl_abort_req;
+
+ if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev)
+ return;
+
+ if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
+ send_tx_flowc_wr(csk);
+ cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
+ }
+
+ cxgbi_sock_set_state(csk, CTP_ABORTING);
+ cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
+ cxgbi_sock_purge_write_queue(csk);
+
+ csk->cpl_abort_req = NULL;
+ req = (struct cpl_abort_req *)skb->head;
+ set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
+ req->cmd = CPL_ABORT_SEND_RST;
+ t4_set_arp_err_handler(skb, csk, abort_arp_failure);
+ INIT_TP_WR(req, csk->tid);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
+ req->rsvd0 = htonl(csk->snd_nxt);
+ req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
+ csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
+ req->rsvd1);
+
+ cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
+}
+
+static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
+{
+ struct sk_buff *skb = csk->cpl_abort_rpl;
+ struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, status %d.\n",
+ csk, csk->state, csk->flags, csk->tid, rst_status);
+
+ csk->cpl_abort_rpl = NULL;
+ set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
+ INIT_TP_WR(rpl, csk->tid);
+ OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
+ rpl->cmd = rst_status;
+ cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
+}
+
+/*
+ * CPL connection rx data ack: host ->
+ * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
+ * credits sent.
+ */
+static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
+{
+ struct sk_buff *skb;
+ struct cpl_rx_data_ack *req;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx,%u, credit %u.\n",
+ csk, csk->state, csk->flags, csk->tid, credits);
+
+ skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
+ if (!skb) {
+ pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
+ return 0;
+ }
+ req = (struct cpl_rx_data_ack *)skb->head;
+
+ set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id);
+ INIT_TP_WR(req, csk->tid);
+ OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
+ csk->tid));
+ req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits)
+ | RX_FORCE_ACK_F);
+ cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
+ return credits;
+}
+
+/*
+ * sgl_len - calculates the size of an SGL of the given capacity
+ * @n: the number of SGL entries
+ * Calculates the number of flits needed for a scatter/gather list that
+ * can hold the given number of entries.
+ */
+static inline unsigned int sgl_len(unsigned int n)
+{
+ n--;
+ return (3 * n) / 2 + (n & 1) + 2;
+}
+
+/*
+ * calc_tx_flits_ofld - calculate # of flits for an offload packet
+ * @skb: the packet
+ *
+ * Returns the number of flits needed for the given offload packet.
+ * These packets are already fully constructed and no additional headers
+ * will be added.
+ */
+static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
+{
+ unsigned int flits, cnt;
+
+ if (is_ofld_imm(skb))
+ return DIV_ROUND_UP(skb->len, 8);
+ flits = skb_transport_offset(skb) / 8;
+ cnt = skb_shinfo(skb)->nr_frags;
+ if (skb_tail_pointer(skb) != skb_transport_header(skb))
+ cnt++;
+ return flits + sgl_len(cnt);
+}
+
+#define FLOWC_WR_NPARAMS_MIN 9
+static inline int tx_flowc_wr_credits(int *nparamsp, int *flowclenp)
+{
+ int nparams, flowclen16, flowclen;
+
+ nparams = FLOWC_WR_NPARAMS_MIN;
+#ifdef CONFIG_CHELSIO_T4_DCB
+ nparams++;
+#endif
+ flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
+ flowclen16 = DIV_ROUND_UP(flowclen, 16);
+ flowclen = flowclen16 * 16;
+ /*
+ * Return the number of 16-byte credits used by the FlowC request.
+ * Pass back the nparams and actual FlowC length if requested.
+ */
+ if (nparamsp)
+ *nparamsp = nparams;
+ if (flowclenp)
+ *flowclenp = flowclen;
+
+ return flowclen16;
+}
+
+static inline int send_tx_flowc_wr(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb;
+ struct fw_flowc_wr *flowc;
+ int nparams, flowclen16, flowclen;
+
+#ifdef CONFIG_CHELSIO_T4_DCB
+ u16 vlan = ((struct l2t_entry *)csk->l2t)->vlan;
+#endif
+ flowclen16 = tx_flowc_wr_credits(&nparams, &flowclen);
+ skb = alloc_wr(flowclen, 0, GFP_ATOMIC);
+ flowc = (struct fw_flowc_wr *)skb->head;
+ flowc->op_to_nparams =
+ htonl(FW_WR_OP_V(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS_V(nparams));
+ flowc->flowid_len16 =
+ htonl(FW_WR_LEN16_V(flowclen16) | FW_WR_FLOWID_V(csk->tid));
+ flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
+ flowc->mnemval[0].val = htonl(csk->cdev->pfvf);
+ flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
+ flowc->mnemval[1].val = htonl(csk->tx_chan);
+ flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
+ flowc->mnemval[2].val = htonl(csk->tx_chan);
+ flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
+ flowc->mnemval[3].val = htonl(csk->rss_qid);
+ flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
+ flowc->mnemval[4].val = htonl(csk->snd_nxt);
+ flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
+ flowc->mnemval[5].val = htonl(csk->rcv_nxt);
+ flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
+ flowc->mnemval[6].val = htonl(csk->snd_win);
+ flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
+ flowc->mnemval[7].val = htonl(csk->advmss);
+ flowc->mnemval[8].mnemonic = 0;
+ flowc->mnemval[8].val = 0;
+ flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
+ if (csk->cdev->skb_iso_txhdr)
+ flowc->mnemval[8].val = cpu_to_be32(CXGBI_MAX_ISO_DATA_IN_SKB);
+ else
+ flowc->mnemval[8].val = cpu_to_be32(16128);
+#ifdef CONFIG_CHELSIO_T4_DCB
+ flowc->mnemval[9].mnemonic = FW_FLOWC_MNEM_DCBPRIO;
+ if (vlan == CPL_L2T_VLAN_NONE) {
+ pr_warn_ratelimited("csk %u without VLAN Tag on DCB Link\n",
+ csk->tid);
+ flowc->mnemval[9].val = cpu_to_be32(0);
+ } else {
+ flowc->mnemval[9].val = cpu_to_be32((vlan & VLAN_PRIO_MASK) >>
+ VLAN_PRIO_SHIFT);
+ }
+#endif
+
+ set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n",
+ csk, csk->tid, 0, csk->tx_chan, csk->rss_qid,
+ csk->snd_nxt, csk->rcv_nxt, csk->snd_win,
+ csk->advmss);
+
+ cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
+
+ return flowclen16;
+}
+
+static void
+cxgb4i_make_tx_iso_cpl(struct sk_buff *skb, struct cpl_tx_data_iso *cpl)
+{
+ struct cxgbi_iso_info *info = (struct cxgbi_iso_info *)skb->head;
+ u32 imm_en = !!(info->flags & CXGBI_ISO_INFO_IMM_ENABLE);
+ u32 fslice = !!(info->flags & CXGBI_ISO_INFO_FSLICE);
+ u32 lslice = !!(info->flags & CXGBI_ISO_INFO_LSLICE);
+ u32 pdu_type = (info->op == ISCSI_OP_SCSI_CMD) ? 0 : 1;
+ u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3;
+
+ cpl->op_to_scsi = cpu_to_be32(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
+ CPL_TX_DATA_ISO_FIRST_V(fslice) |
+ CPL_TX_DATA_ISO_LAST_V(lslice) |
+ CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
+ CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
+ CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
+ CPL_TX_DATA_ISO_IMMEDIATE_V(imm_en) |
+ CPL_TX_DATA_ISO_SCSI_V(pdu_type));
+
+ cpl->ahs_len = info->ahs;
+ cpl->mpdu = cpu_to_be16(DIV_ROUND_UP(info->mpdu, 4));
+ cpl->burst_size = cpu_to_be32(info->burst_size);
+ cpl->len = cpu_to_be32(info->len);
+ cpl->reserved2_seglen_offset =
+ cpu_to_be32(CPL_TX_DATA_ISO_SEGLEN_OFFSET_V(info->segment_offset));
+ cpl->datasn_offset = cpu_to_be32(info->datasn_offset);
+ cpl->buffer_offset = cpu_to_be32(info->buffer_offset);
+ cpl->reserved3 = cpu_to_be32(0);
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "iso: flags 0x%x, op %u, ahs %u, num_pdu %u, mpdu %u, "
+ "burst_size %u, iso_len %u\n",
+ info->flags, info->op, info->ahs, info->num_pdu,
+ info->mpdu, info->burst_size << 2, info->len);
+}
+
+static void
+cxgb4i_make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, int dlen,
+ int len, u32 credits, int compl)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct fw_ofld_tx_data_wr *req;
+ struct cpl_tx_data_iso *cpl;
+ u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3;
+ u32 wr_ulp_mode = 0;
+ u32 hdr_size = sizeof(*req);
+ u32 opcode = FW_OFLD_TX_DATA_WR;
+ u32 immlen = 0;
+ u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
+ T6_TX_FORCE_F;
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) {
+ hdr_size += sizeof(struct cpl_tx_data_iso);
+ opcode = FW_ISCSI_TX_DATA_WR;
+ immlen += sizeof(struct cpl_tx_data_iso);
+ submode |= 8;
+ }
+
+ if (is_ofld_imm(skb))
+ immlen += dlen;
+
+ req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, hdr_size);
+ req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
+ FW_WR_COMPL_V(compl) |
+ FW_WR_IMMDLEN_V(immlen));
+ req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
+ FW_WR_LEN16_V(credits));
+ req->plen = cpu_to_be32(len);
+ cpl = (struct cpl_tx_data_iso *)(req + 1);
+
+ if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)))
+ cxgb4i_make_tx_iso_cpl(skb, cpl);
+
+ if (submode)
+ wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) |
+ FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
+
+ req->tunnel_to_proxy = cpu_to_be32(wr_ulp_mode | force |
+ FW_OFLD_TX_DATA_WR_SHOVE_V(1U));
+
+ if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT))
+ cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
+}
+
+static void arp_failure_skb_discard(void *handle, struct sk_buff *skb)
+{
+ kfree_skb(skb);
+}
+
+static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
+{
+ int total_size = 0;
+ struct sk_buff *skb;
+
+ if (unlikely(csk->state < CTP_ESTABLISHED ||
+ csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK |
+ 1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ return 0;
+ }
+
+ while (csk->wr_cred && ((skb = skb_peek(&csk->write_queue)) != NULL)) {
+ struct cxgbi_iso_info *iso_cpl;
+ u32 dlen = skb->len;
+ u32 len = skb->len;
+ u32 iso_cpl_len = 0;
+ u32 flowclen16 = 0;
+ u32 credits_needed;
+ u32 num_pdu = 1, hdr_len;
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))
+ iso_cpl_len = sizeof(struct cpl_tx_data_iso);
+
+ if (is_ofld_imm(skb))
+ credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
+ else
+ credits_needed =
+ DIV_ROUND_UP((8 * calc_tx_flits_ofld(skb)) +
+ iso_cpl_len, 16);
+
+ if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))
+ credits_needed +=
+ DIV_ROUND_UP(sizeof(struct fw_ofld_tx_data_wr), 16);
+
+ /*
+ * Assumes the initial credits is large enough to support
+ * fw_flowc_wr plus largest possible first payload
+ */
+ if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
+ flowclen16 = send_tx_flowc_wr(csk);
+ csk->wr_cred -= flowclen16;
+ csk->wr_una_cred += flowclen16;
+ cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
+ }
+
+ if (csk->wr_cred < credits_needed) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p, skb %u/%u, wr %d < %u.\n",
+ csk, skb->len, skb->data_len,
+ credits_needed, csk->wr_cred);
+
+ csk->no_tx_credits++;
+ break;
+ }
+
+ csk->no_tx_credits = 0;
+
+ __skb_unlink(skb, &csk->write_queue);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
+ skb->csum = (__force __wsum)(credits_needed + flowclen16);
+ csk->wr_cred -= credits_needed;
+ csk->wr_una_cred += credits_needed;
+ cxgbi_sock_enqueue_wr(csk, skb);
+
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
+ csk, skb->len, skb->data_len, credits_needed,
+ csk->wr_cred, csk->wr_una_cred);
+
+ if (!req_completion &&
+ ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
+ after(csk->write_seq, (csk->snd_una + csk->snd_win / 2))))
+ req_completion = 1;
+
+ if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
+ u32 ulp_mode = cxgbi_skcb_tx_ulp_mode(skb);
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) {
+ iso_cpl = (struct cxgbi_iso_info *)skb->head;
+ num_pdu = iso_cpl->num_pdu;
+ hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb);
+ len += (cxgbi_ulp_extra_len(ulp_mode) * num_pdu) +
+ (hdr_len * (num_pdu - 1));
+ } else {
+ len += cxgbi_ulp_extra_len(ulp_mode);
+ }
+
+ cxgb4i_make_tx_data_wr(csk, skb, dlen, len,
+ credits_needed, req_completion);
+ csk->snd_nxt += len;
+ cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
+ } else if (cxgbi_skcb_test_flag(skb, SKCBF_TX_FLAG_COMPL) &&
+ (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
+ struct cpl_close_con_req *req =
+ (struct cpl_close_con_req *)skb->data;
+
+ req->wr.wr_hi |= cpu_to_be32(FW_WR_COMPL_F);
+ }
+
+ total_size += skb->truesize;
+ t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",
+ csk, csk->state, csk->flags, csk->tid, skb, len);
+ cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
+ }
+ return total_size;
+}
+
+static inline void free_atid(struct cxgbi_sock *csk)
+{
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
+
+ if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
+ cxgb4_free_atid(lldi->tids, csk->atid);
+ cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
+ cxgbi_sock_put(csk);
+ }
+}
+
+static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data;
+ unsigned short tcp_opt = ntohs(req->tcp_opt);
+ unsigned int tid = GET_TID(req);
+ unsigned int atid = TID_TID_G(ntohl(req->tos_atid));
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+ u32 rcv_isn = be32_to_cpu(req->rcv_isn);
+
+ csk = lookup_atid(t, atid);
+ if (unlikely(!csk)) {
+ pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid, cdev);
+ goto rel_skb;
+ }
+
+ if (csk->atid != atid) {
+ pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n",
+ atid, csk, csk->state, csk->flags, csk->tid, csk->atid);
+ goto rel_skb;
+ }
+
+ pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
+ (&csk->saddr), (&csk->daddr),
+ atid, tid, csk, csk->state, csk->flags, rcv_isn);
+
+ module_put(cdev->owner);
+
+ cxgbi_sock_get(csk);
+ csk->tid = tid;
+ cxgb4_insert_tid(lldi->tids, csk, tid, csk->csk_family);
+ cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
+
+ free_atid(csk);
+
+ spin_lock_bh(&csk->lock);
+ if (unlikely(csk->state != CTP_ACTIVE_OPEN))
+ pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ if (csk->retry_timer.function) {
+ del_timer(&csk->retry_timer);
+ csk->retry_timer.function = NULL;
+ }
+
+ csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
+ /*
+ * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
+ * pass through opt0.
+ */
+ if (csk->rcv_win > (RCV_BUFSIZ_MASK << 10))
+ csk->rcv_wup -= csk->rcv_win - (RCV_BUFSIZ_MASK << 10);
+
+ csk->advmss = lldi->mtus[TCPOPT_MSS_G(tcp_opt)] - 40;
+ if (TCPOPT_TSTAMP_G(tcp_opt))
+ csk->advmss -= 12;
+ if (csk->advmss < 128)
+ csk->advmss = 128;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p, mss_idx %u, advmss %u.\n",
+ csk, TCPOPT_MSS_G(tcp_opt), csk->advmss);
+
+ cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
+
+ if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
+ send_abort_req(csk);
+ else {
+ if (skb_queue_len(&csk->write_queue))
+ push_tx_frames(csk, 0);
+ cxgbi_conn_tx_open(csk);
+ }
+ spin_unlock_bh(&csk->lock);
+
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static int act_open_rpl_status_to_errno(int status)
+{
+ switch (status) {
+ case CPL_ERR_CONN_RESET:
+ return -ECONNREFUSED;
+ case CPL_ERR_ARP_MISS:
+ return -EHOSTUNREACH;
+ case CPL_ERR_CONN_TIMEDOUT:
+ return -ETIMEDOUT;
+ case CPL_ERR_TCAM_FULL:
+ return -ENOMEM;
+ case CPL_ERR_CONN_EXIST:
+ return -EADDRINUSE;
+ default:
+ return -EIO;
+ }
+}
+
+static void csk_act_open_retry_timer(struct timer_list *t)
+{
+ struct sk_buff *skb = NULL;
+ struct cxgbi_sock *csk = from_timer(csk, t, retry_timer);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
+ void (*send_act_open_func)(struct cxgbi_sock *, struct sk_buff *,
+ struct l2t_entry *);
+ int t4 = is_t4(lldi->adapter_type), size, size6;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+
+ if (t4) {
+ size = sizeof(struct cpl_act_open_req);
+ size6 = sizeof(struct cpl_act_open_req6);
+ } else {
+ size = sizeof(struct cpl_t5_act_open_req);
+ size6 = sizeof(struct cpl_t5_act_open_req6);
+ }
+
+ if (csk->csk_family == AF_INET) {
+ send_act_open_func = send_act_open_req;
+ skb = alloc_wr(size, 0, GFP_ATOMIC);
+#if IS_ENABLED(CONFIG_IPV6)
+ } else {
+ send_act_open_func = send_act_open_req6;
+ skb = alloc_wr(size6, 0, GFP_ATOMIC);
+#endif
+ }
+
+ if (!skb)
+ cxgbi_sock_fail_act_open(csk, -ENOMEM);
+ else {
+ skb->sk = (struct sock *)csk;
+ t4_set_arp_err_handler(skb, csk,
+ cxgbi_sock_act_open_req_arp_failure);
+ send_act_open_func(csk, skb, csk->l2t);
+ }
+
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+
+}
+
+static inline bool is_neg_adv(unsigned int status)
+{
+ return status == CPL_ERR_RTX_NEG_ADVICE ||
+ status == CPL_ERR_KEEPALV_NEG_ADVICE ||
+ status == CPL_ERR_PERSIST_NEG_ADVICE;
+}
+
+static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data;
+ unsigned int tid = GET_TID(rpl);
+ unsigned int atid =
+ TID_TID_G(AOPEN_ATID_G(be32_to_cpu(rpl->atid_status)));
+ unsigned int status = AOPEN_STATUS_G(be32_to_cpu(rpl->atid_status));
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+
+ csk = lookup_atid(t, atid);
+ if (unlikely(!csk)) {
+ pr_err("NO matching conn. atid %u, tid %u.\n", atid, tid);
+ goto rel_skb;
+ }
+
+ pr_info_ipaddr("tid %u/%u, status %u.\n"
+ "csk 0x%p,%u,0x%lx. ", (&csk->saddr), (&csk->daddr),
+ atid, tid, status, csk, csk->state, csk->flags);
+
+ if (is_neg_adv(status))
+ goto rel_skb;
+
+ module_put(cdev->owner);
+
+ if (status && status != CPL_ERR_TCAM_FULL &&
+ status != CPL_ERR_CONN_EXIST &&
+ status != CPL_ERR_ARP_MISS)
+ cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl),
+ csk->csk_family);
+
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+
+ if (status == CPL_ERR_CONN_EXIST &&
+ csk->retry_timer.function != csk_act_open_retry_timer) {
+ csk->retry_timer.function = csk_act_open_retry_timer;
+ mod_timer(&csk->retry_timer, jiffies + HZ / 2);
+ } else
+ cxgbi_sock_fail_act_open(csk,
+ act_open_rpl_status_to_errno(status));
+
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data;
+ unsigned int tid = GET_TID(req);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+
+ csk = lookup_tid(t, tid);
+ if (unlikely(!csk)) {
+ pr_err("can't find connection for tid %u.\n", tid);
+ goto rel_skb;
+ }
+ pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
+ (&csk->saddr), (&csk->daddr),
+ csk, csk->state, csk->flags, csk->tid);
+ cxgbi_sock_rcv_peer_close(csk);
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data;
+ unsigned int tid = GET_TID(rpl);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+
+ csk = lookup_tid(t, tid);
+ if (unlikely(!csk)) {
+ pr_err("can't find connection for tid %u.\n", tid);
+ goto rel_skb;
+ }
+ pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
+ (&csk->saddr), (&csk->daddr),
+ csk, csk->state, csk->flags, csk->tid);
+ cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
+ int *need_rst)
+{
+ switch (abort_reason) {
+ case CPL_ERR_BAD_SYN:
+ case CPL_ERR_CONN_RESET:
+ return csk->state > CTP_ESTABLISHED ?
+ -EPIPE : -ECONNRESET;
+ case CPL_ERR_XMIT_TIMEDOUT:
+ case CPL_ERR_PERSIST_TIMEDOUT:
+ case CPL_ERR_FINWAIT2_TIMEDOUT:
+ case CPL_ERR_KEEPALIVE_TIMEDOUT:
+ return -ETIMEDOUT;
+ default:
+ return -EIO;
+ }
+}
+
+static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data;
+ unsigned int tid = GET_TID(req);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+ int rst_status = CPL_ABORT_NO_RST;
+
+ csk = lookup_tid(t, tid);
+ if (unlikely(!csk)) {
+ pr_err("can't find connection for tid %u.\n", tid);
+ goto rel_skb;
+ }
+
+ pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
+ (&csk->saddr), (&csk->daddr),
+ csk, csk->state, csk->flags, csk->tid, req->status);
+
+ if (is_neg_adv(req->status))
+ goto rel_skb;
+
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+
+ cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
+
+ if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
+ send_tx_flowc_wr(csk);
+ cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
+ }
+
+ cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
+ cxgbi_sock_set_state(csk, CTP_ABORTING);
+
+ send_abort_rpl(csk, rst_status);
+
+ if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
+ csk->err = abort_status_to_errno(csk, req->status, &rst_status);
+ cxgbi_sock_closed(csk);
+ }
+
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data;
+ unsigned int tid = GET_TID(rpl);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+
+ csk = lookup_tid(t, tid);
+ if (!csk)
+ goto rel_skb;
+
+ pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
+ (&csk->saddr), (&csk->daddr), csk,
+ csk->state, csk->flags, csk->tid, rpl->status);
+
+ if (rpl->status == CPL_ERR_ABORT_FAILED)
+ goto rel_skb;
+
+ cxgbi_sock_rcv_abort_rpl(csk);
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static void do_rx_data(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_rx_data *cpl = (struct cpl_rx_data *)skb->data;
+ unsigned int tid = GET_TID(cpl);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+
+ csk = lookup_tid(t, tid);
+ if (!csk) {
+ pr_err("can't find connection for tid %u.\n", tid);
+ } else {
+ /* not expecting this, reset the connection. */
+ pr_err("csk 0x%p, tid %u, rcv cpl_rx_data.\n", csk, tid);
+ spin_lock_bh(&csk->lock);
+ send_abort_req(csk);
+ spin_unlock_bh(&csk->lock);
+ }
+ __kfree_skb(skb);
+}
+
+static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data;
+ unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp);
+ unsigned int tid = GET_TID(cpl);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+
+ csk = lookup_tid(t, tid);
+ if (unlikely(!csk)) {
+ pr_err("can't find conn. for tid %u.\n", tid);
+ goto rel_skb;
+ }
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n",
+ csk, csk->state, csk->flags, csk->tid, skb, skb->len,
+ pdu_len_ddp);
+
+ spin_lock_bh(&csk->lock);
+
+ if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, bad state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ if (csk->state != CTP_ABORTING)
+ goto abort_conn;
+ else
+ goto discard;
+ }
+
+ cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq);
+ cxgbi_skcb_flags(skb) = 0;
+
+ skb_reset_transport_header(skb);
+ __skb_pull(skb, sizeof(*cpl));
+ __pskb_trim(skb, ntohs(cpl->len));
+
+ if (!csk->skb_ulp_lhdr) {
+ unsigned char *bhs;
+ unsigned int hlen, dlen, plen;
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n",
+ csk, csk->state, csk->flags, csk->tid, skb);
+ csk->skb_ulp_lhdr = skb;
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR);
+
+ if ((CHELSIO_CHIP_VERSION(lldi->adapter_type) <= CHELSIO_T5) &&
+ (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt)) {
+ pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n",
+ csk->tid, cxgbi_skcb_tcp_seq(skb),
+ csk->rcv_nxt);
+ goto abort_conn;
+ }
+
+ bhs = skb->data;
+ hlen = ntohs(cpl->len);
+ dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF;
+
+ plen = ISCSI_PDU_LEN_G(pdu_len_ddp);
+ if (is_t4(lldi->adapter_type))
+ plen -= 40;
+
+ if ((hlen + dlen) != plen) {
+ pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len "
+ "mismatch %u != %u + %u, seq 0x%x.\n",
+ csk->tid, plen, hlen, dlen,
+ cxgbi_skcb_tcp_seq(skb));
+ goto abort_conn;
+ }
+
+ cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3);
+ if (dlen)
+ cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len;
+ csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n",
+ csk, skb, *bhs, hlen, dlen,
+ ntohl(*((unsigned int *)(bhs + 16))),
+ ntohl(*((unsigned int *)(bhs + 24))));
+
+ } else {
+ struct sk_buff *lskb = csk->skb_ulp_lhdr;
+
+ cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA);
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n",
+ csk, csk->state, csk->flags, skb, lskb);
+ }
+
+ __skb_queue_tail(&csk->receive_queue, skb);
+ spin_unlock_bh(&csk->lock);
+ return;
+
+abort_conn:
+ send_abort_req(csk);
+discard:
+ spin_unlock_bh(&csk->lock);
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static void do_rx_iscsi_data(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data;
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+ struct sk_buff *lskb;
+ u32 tid = GET_TID(cpl);
+ u16 pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp);
+
+ csk = lookup_tid(t, tid);
+ if (unlikely(!csk)) {
+ pr_err("can't find conn. for tid %u.\n", tid);
+ goto rel_skb;
+ }
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n",
+ csk, csk->state, csk->flags, csk->tid, skb,
+ skb->len, pdu_len_ddp);
+
+ spin_lock_bh(&csk->lock);
+
+ if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, bad state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ if (csk->state != CTP_ABORTING)
+ goto abort_conn;
+ else
+ goto discard;
+ }
+
+ cxgbi_skcb_tcp_seq(skb) = be32_to_cpu(cpl->seq);
+ cxgbi_skcb_flags(skb) = 0;
+
+ skb_reset_transport_header(skb);
+ __skb_pull(skb, sizeof(*cpl));
+ __pskb_trim(skb, ntohs(cpl->len));
+
+ if (!csk->skb_ulp_lhdr)
+ csk->skb_ulp_lhdr = skb;
+
+ lskb = csk->skb_ulp_lhdr;
+ cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n",
+ csk, csk->state, csk->flags, skb, lskb);
+
+ __skb_queue_tail(&csk->receive_queue, skb);
+ spin_unlock_bh(&csk->lock);
+ return;
+
+abort_conn:
+ send_abort_req(csk);
+discard:
+ spin_unlock_bh(&csk->lock);
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static void
+cxgb4i_process_ddpvld(struct cxgbi_sock *csk,
+ struct sk_buff *skb, u32 ddpvld)
+{
+ if (ddpvld & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) {
+ pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n",
+ csk, skb, ddpvld, cxgbi_skcb_flags(skb));
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);
+ }
+
+ if (ddpvld & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) {
+ pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n",
+ csk, skb, ddpvld, cxgbi_skcb_flags(skb));
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);
+ }
+
+ if (ddpvld & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) {
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n",
+ csk, skb, ddpvld);
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);
+ }
+
+ if ((ddpvld & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) &&
+ !cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) {
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n",
+ csk, skb, ddpvld);
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);
+ }
+}
+
+static void do_rx_data_ddp(struct cxgbi_device *cdev,
+ struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct sk_buff *lskb;
+ struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data;
+ unsigned int tid = GET_TID(rpl);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+ u32 ddpvld = be32_to_cpu(rpl->ddpvld);
+
+ csk = lookup_tid(t, tid);
+ if (unlikely(!csk)) {
+ pr_err("can't find connection for tid %u.\n", tid);
+ goto rel_skb;
+ }
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n",
+ csk, csk->state, csk->flags, skb, ddpvld, csk->skb_ulp_lhdr);
+
+ spin_lock_bh(&csk->lock);
+
+ if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, bad state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ if (csk->state != CTP_ABORTING)
+ goto abort_conn;
+ else
+ goto discard;
+ }
+
+ if (!csk->skb_ulp_lhdr) {
+ pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk->tid);
+ goto abort_conn;
+ }
+
+ lskb = csk->skb_ulp_lhdr;
+ csk->skb_ulp_lhdr = NULL;
+
+ cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc);
+
+ if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb))
+ pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n",
+ csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb));
+
+ cxgb4i_process_ddpvld(csk, lskb, ddpvld);
+
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, lskb 0x%p, f 0x%lx.\n",
+ csk, lskb, cxgbi_skcb_flags(lskb));
+
+ cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS);
+ cxgbi_conn_pdu_ready(csk);
+ spin_unlock_bh(&csk->lock);
+ goto rel_skb;
+
+abort_conn:
+ send_abort_req(csk);
+discard:
+ spin_unlock_bh(&csk->lock);
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static void
+do_rx_iscsi_cmp(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_rx_iscsi_cmp *rpl = (struct cpl_rx_iscsi_cmp *)skb->data;
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+ struct sk_buff *data_skb = NULL;
+ u32 tid = GET_TID(rpl);
+ u32 ddpvld = be32_to_cpu(rpl->ddpvld);
+ u32 seq = be32_to_cpu(rpl->seq);
+ u16 pdu_len_ddp = be16_to_cpu(rpl->pdu_len_ddp);
+
+ csk = lookup_tid(t, tid);
+ if (unlikely(!csk)) {
+ pr_err("can't find connection for tid %u.\n", tid);
+ goto rel_skb;
+ }
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p, len %u, "
+ "pdu_len_ddp %u, status %u.\n",
+ csk, csk->state, csk->flags, skb, ddpvld, csk->skb_ulp_lhdr,
+ ntohs(rpl->len), pdu_len_ddp, rpl->status);
+
+ spin_lock_bh(&csk->lock);
+
+ if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, bad state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ if (csk->state != CTP_ABORTING)
+ goto abort_conn;
+ else
+ goto discard;
+ }
+
+ cxgbi_skcb_tcp_seq(skb) = seq;
+ cxgbi_skcb_flags(skb) = 0;
+ cxgbi_skcb_rx_pdulen(skb) = 0;
+
+ skb_reset_transport_header(skb);
+ __skb_pull(skb, sizeof(*rpl));
+ __pskb_trim(skb, be16_to_cpu(rpl->len));
+
+ csk->rcv_nxt = seq + pdu_len_ddp;
+
+ if (csk->skb_ulp_lhdr) {
+ data_skb = skb_peek(&csk->receive_queue);
+ if (!data_skb ||
+ !cxgbi_skcb_test_flag(data_skb, SKCBF_RX_DATA)) {
+ pr_err("Error! freelist data not found 0x%p, tid %u\n",
+ data_skb, tid);
+
+ goto abort_conn;
+ }
+ __skb_unlink(data_skb, &csk->receive_queue);
+
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA);
+
+ __skb_queue_tail(&csk->receive_queue, skb);
+ __skb_queue_tail(&csk->receive_queue, data_skb);
+ } else {
+ __skb_queue_tail(&csk->receive_queue, skb);
+ }
+
+ csk->skb_ulp_lhdr = NULL;
+
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR);
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);
+ cxgbi_skcb_set_flag(skb, SKCBF_RX_ISCSI_COMPL);
+ cxgbi_skcb_rx_ddigest(skb) = be32_to_cpu(rpl->ulp_crc);
+
+ cxgb4i_process_ddpvld(csk, skb, ddpvld);
+
+ log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, skb 0x%p, f 0x%lx.\n",
+ csk, skb, cxgbi_skcb_flags(skb));
+
+ cxgbi_conn_pdu_ready(csk);
+ spin_unlock_bh(&csk->lock);
+
+ return;
+
+abort_conn:
+ send_abort_req(csk);
+discard:
+ spin_unlock_bh(&csk->lock);
+rel_skb:
+ __kfree_skb(skb);
+}
+
+static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk;
+ struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data;
+ unsigned int tid = GET_TID(rpl);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+
+ csk = lookup_tid(t, tid);
+ if (unlikely(!csk))
+ pr_err("can't find connection for tid %u.\n", tid);
+ else {
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una),
+ rpl->seq_vld);
+ }
+ __kfree_skb(skb);
+}
+
+static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
+{
+ struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data;
+ unsigned int tid = GET_TID(rpl);
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct tid_info *t = lldi->tids;
+ struct cxgbi_sock *csk;
+
+ csk = lookup_tid(t, tid);
+ if (!csk) {
+ pr_err("can't find conn. for tid %u.\n", tid);
+ return;
+ }
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,%lx,%u, status 0x%x.\n",
+ csk, csk->state, csk->flags, csk->tid, rpl->status);
+
+ if (rpl->status != CPL_ERR_NONE) {
+ pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n",
+ csk, tid, rpl->status);
+ csk->err = -EINVAL;
+ }
+
+ complete(&csk->cmpl);
+
+ __kfree_skb(skb);
+}
+
+static int alloc_cpls(struct cxgbi_sock *csk)
+{
+ csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req),
+ 0, GFP_KERNEL);
+ if (!csk->cpl_close)
+ return -ENOMEM;
+
+ csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req),
+ 0, GFP_KERNEL);
+ if (!csk->cpl_abort_req)
+ goto free_cpls;
+
+ csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl),
+ 0, GFP_KERNEL);
+ if (!csk->cpl_abort_rpl)
+ goto free_cpls;
+ return 0;
+
+free_cpls:
+ cxgbi_sock_free_cpl_skbs(csk);
+ return -ENOMEM;
+}
+
+static inline void l2t_put(struct cxgbi_sock *csk)
+{
+ if (csk->l2t) {
+ cxgb4_l2t_release(csk->l2t);
+ csk->l2t = NULL;
+ cxgbi_sock_put(csk);
+ }
+}
+
+static void release_offload_resources(struct cxgbi_sock *csk)
+{
+ struct cxgb4_lld_info *lldi;
+#if IS_ENABLED(CONFIG_IPV6)
+ struct net_device *ndev = csk->cdev->ports[csk->port_id];
+#endif
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ cxgbi_sock_free_cpl_skbs(csk);
+ cxgbi_sock_purge_write_queue(csk);
+ if (csk->wr_cred != csk->wr_max_cred) {
+ cxgbi_sock_purge_wr_queue(csk);
+ cxgbi_sock_reset_wr_list(csk);
+ }
+
+ l2t_put(csk);
+#if IS_ENABLED(CONFIG_IPV6)
+ if (csk->csk_family == AF_INET6)
+ cxgb4_clip_release(ndev,
+ (const u32 *)&csk->saddr6.sin6_addr, 1);
+#endif
+
+ if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
+ free_atid(csk);
+ else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
+ lldi = cxgbi_cdev_priv(csk->cdev);
+ cxgb4_remove_tid(lldi->tids, 0, csk->tid,
+ csk->csk_family);
+ cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
+ cxgbi_sock_put(csk);
+ }
+ csk->dst = NULL;
+}
+
+#ifdef CONFIG_CHELSIO_T4_DCB
+static inline u8 get_iscsi_dcb_state(struct net_device *ndev)
+{
+ return ndev->dcbnl_ops->getstate(ndev);
+}
+
+static int select_priority(int pri_mask)
+{
+ if (!pri_mask)
+ return 0;
+ return (ffs(pri_mask) - 1);
+}
+
+static u8 get_iscsi_dcb_priority(struct net_device *ndev)
+{
+ int rv;
+ u8 caps;
+
+ struct dcb_app iscsi_dcb_app = {
+ .protocol = 3260
+ };
+
+ rv = (int)ndev->dcbnl_ops->getcap(ndev, DCB_CAP_ATTR_DCBX, &caps);
+ if (rv)
+ return 0;
+
+ if (caps & DCB_CAP_DCBX_VER_IEEE) {
+ iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_STREAM;
+ rv = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app);
+ if (!rv) {
+ iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_ANY;
+ rv = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app);
+ }
+ } else if (caps & DCB_CAP_DCBX_VER_CEE) {
+ iscsi_dcb_app.selector = DCB_APP_IDTYPE_PORTNUM;
+ rv = dcb_getapp(ndev, &iscsi_dcb_app);
+ }
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "iSCSI priority is set to %u\n", select_priority(rv));
+ return select_priority(rv);
+}
+#endif
+
+static int init_act_open(struct cxgbi_sock *csk)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct net_device *ndev = cdev->ports[csk->port_id];
+ struct sk_buff *skb = NULL;
+ struct neighbour *n = NULL;
+ void *daddr;
+ unsigned int step;
+ unsigned int rxq_idx;
+ unsigned int size, size6;
+ unsigned int linkspeed;
+ unsigned int rcv_winf, snd_winf;
+#ifdef CONFIG_CHELSIO_T4_DCB
+ u8 priority = 0;
+#endif
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, csk->state, csk->flags, csk->tid);
+
+ if (csk->csk_family == AF_INET)
+ daddr = &csk->daddr.sin_addr.s_addr;
+#if IS_ENABLED(CONFIG_IPV6)
+ else if (csk->csk_family == AF_INET6)
+ daddr = &csk->daddr6.sin6_addr;
+#endif
+ else {
+ pr_err("address family 0x%x not supported\n", csk->csk_family);
+ goto rel_resource;
+ }
+
+ n = dst_neigh_lookup(csk->dst, daddr);
+
+ if (!n) {
+ pr_err("%s, can't get neighbour of csk->dst.\n", ndev->name);
+ goto rel_resource;
+ }
+
+ if (!(n->nud_state & NUD_VALID))
+ neigh_event_send(n, NULL);
+
+ csk->atid = cxgb4_alloc_atid(lldi->tids, csk);
+ if (csk->atid < 0) {
+ pr_err("%s, NO atid available.\n", ndev->name);
+ goto rel_resource_without_clip;
+ }
+ cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
+ cxgbi_sock_get(csk);
+
+#ifdef CONFIG_CHELSIO_T4_DCB
+ if (get_iscsi_dcb_state(ndev))
+ priority = get_iscsi_dcb_priority(ndev);
+
+ csk->dcb_priority = priority;
+ csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, priority);
+#else
+ csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, 0);
+#endif
+ if (!csk->l2t) {
+ pr_err("%s, cannot alloc l2t.\n", ndev->name);
+ goto rel_resource_without_clip;
+ }
+ cxgbi_sock_get(csk);
+
+#if IS_ENABLED(CONFIG_IPV6)
+ if (csk->csk_family == AF_INET6)
+ cxgb4_clip_get(ndev, (const u32 *)&csk->saddr6.sin6_addr, 1);
+#endif
+
+ if (is_t4(lldi->adapter_type)) {
+ size = sizeof(struct cpl_act_open_req);
+ size6 = sizeof(struct cpl_act_open_req6);
+ } else if (is_t5(lldi->adapter_type)) {
+ size = sizeof(struct cpl_t5_act_open_req);
+ size6 = sizeof(struct cpl_t5_act_open_req6);
+ } else {
+ size = sizeof(struct cpl_t6_act_open_req);
+ size6 = sizeof(struct cpl_t6_act_open_req6);
+ }
+
+ if (csk->csk_family == AF_INET)
+ skb = alloc_wr(size, 0, GFP_NOIO);
+#if IS_ENABLED(CONFIG_IPV6)
+ else
+ skb = alloc_wr(size6, 0, GFP_NOIO);
+#endif
+
+ if (!skb)
+ goto rel_resource;
+ skb->sk = (struct sock *)csk;
+ t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure);
+
+ if (!csk->mtu)
+ csk->mtu = dst_mtu(csk->dst);
+ cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx);
+ csk->tx_chan = cxgb4_port_chan(ndev);
+ csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
+ step = lldi->ntxq / lldi->nchan;
+ csk->txq_idx = cxgb4_port_idx(ndev) * step;
+ step = lldi->nrxq / lldi->nchan;
+ rxq_idx = (cxgb4_port_idx(ndev) * step) + (cdev->rxq_idx_cntr % step);
+ cdev->rxq_idx_cntr++;
+ csk->rss_qid = lldi->rxq_ids[rxq_idx];
+ linkspeed = ((struct port_info *)netdev_priv(ndev))->link_cfg.speed;
+ csk->snd_win = cxgb4i_snd_win;
+ csk->rcv_win = cxgb4i_rcv_win;
+ if (cxgb4i_rcv_win <= 0) {
+ csk->rcv_win = CXGB4I_DEFAULT_10G_RCV_WIN;
+ rcv_winf = linkspeed / SPEED_10000;
+ if (rcv_winf)
+ csk->rcv_win *= rcv_winf;
+ }
+ if (cxgb4i_snd_win <= 0) {
+ csk->snd_win = CXGB4I_DEFAULT_10G_SND_WIN;
+ snd_winf = linkspeed / SPEED_10000;
+ if (snd_winf)
+ csk->snd_win *= snd_winf;
+ }
+ csk->wr_cred = lldi->wr_cred -
+ DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16);
+ csk->wr_max_cred = csk->wr_cred;
+ csk->wr_una_cred = 0;
+ cxgbi_sock_reset_wr_list(csk);
+ csk->err = 0;
+
+ pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n",
+ (&csk->saddr), (&csk->daddr), csk, csk->state,
+ csk->flags, csk->tx_chan, csk->txq_idx, csk->rss_qid,
+ csk->mtu, csk->mss_idx, csk->smac_idx);
+
+ /* must wait for either a act_open_rpl or act_open_establish */
+ if (!try_module_get(cdev->owner)) {
+ pr_err("%s, try_module_get failed.\n", ndev->name);
+ goto rel_resource;
+ }
+
+ cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
+ if (csk->csk_family == AF_INET)
+ send_act_open_req(csk, skb, csk->l2t);
+#if IS_ENABLED(CONFIG_IPV6)
+ else
+ send_act_open_req6(csk, skb, csk->l2t);
+#endif
+ neigh_release(n);
+
+ return 0;
+
+rel_resource:
+#if IS_ENABLED(CONFIG_IPV6)
+ if (csk->csk_family == AF_INET6)
+ cxgb4_clip_release(ndev,
+ (const u32 *)&csk->saddr6.sin6_addr, 1);
+#endif
+rel_resource_without_clip:
+ if (n)
+ neigh_release(n);
+ if (skb)
+ __kfree_skb(skb);
+ return -EINVAL;
+}
+
+static cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = {
+ [CPL_ACT_ESTABLISH] = do_act_establish,
+ [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
+ [CPL_PEER_CLOSE] = do_peer_close,
+ [CPL_ABORT_REQ_RSS] = do_abort_req_rss,
+ [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss,
+ [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
+ [CPL_FW4_ACK] = do_fw4_ack,
+ [CPL_ISCSI_HDR] = do_rx_iscsi_hdr,
+ [CPL_ISCSI_DATA] = do_rx_iscsi_data,
+ [CPL_SET_TCB_RPL] = do_set_tcb_rpl,
+ [CPL_RX_DATA_DDP] = do_rx_data_ddp,
+ [CPL_RX_ISCSI_DDP] = do_rx_data_ddp,
+ [CPL_RX_ISCSI_CMP] = do_rx_iscsi_cmp,
+ [CPL_RX_DATA] = do_rx_data,
+};
+
+static int cxgb4i_ofld_init(struct cxgbi_device *cdev)
+{
+ int rc;
+
+ if (cxgb4i_max_connect > CXGB4I_MAX_CONN)
+ cxgb4i_max_connect = CXGB4I_MAX_CONN;
+
+ rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base,
+ cxgb4i_max_connect);
+ if (rc < 0)
+ return rc;
+
+ cdev->csk_release_offload_resources = release_offload_resources;
+ cdev->csk_push_tx_frames = push_tx_frames;
+ cdev->csk_send_abort_req = send_abort_req;
+ cdev->csk_send_close_req = send_close_req;
+ cdev->csk_send_rx_credits = send_rx_credits;
+ cdev->csk_alloc_cpls = alloc_cpls;
+ cdev->csk_init_act_open = init_act_open;
+
+ pr_info("cdev 0x%p, offload up, added.\n", cdev);
+ return 0;
+}
+
+static inline void
+ulp_mem_io_set_hdr(struct cxgbi_device *cdev,
+ struct ulp_mem_io *req,
+ unsigned int wr_len, unsigned int dlen,
+ unsigned int pm_addr,
+ int tid)
+{
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1);
+
+ INIT_ULPTX_WR(req, wr_len, 0, tid);
+ req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) |
+ FW_WR_ATOMIC_V(0));
+ req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
+ ULP_MEMIO_ORDER_V(is_t4(lldi->adapter_type)) |
+ T5_ULP_MEMIO_IMM_V(!is_t4(lldi->adapter_type)));
+ req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5));
+ req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5));
+ req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16));
+
+ idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
+ idata->len = htonl(dlen);
+}
+
+static struct sk_buff *
+ddp_ppod_init_idata(struct cxgbi_device *cdev,
+ struct cxgbi_ppm *ppm,
+ unsigned int idx, unsigned int npods,
+ unsigned int tid)
+{
+ unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
+ unsigned int dlen = npods << PPOD_SIZE_SHIFT;
+ unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
+ sizeof(struct ulptx_idata) + dlen, 16);
+ struct sk_buff *skb = alloc_wr(wr_len, 0, GFP_ATOMIC);
+
+ if (!skb) {
+ pr_err("%s: %s idx %u, npods %u, OOM.\n",
+ __func__, ppm->ndev->name, idx, npods);
+ return NULL;
+ }
+
+ ulp_mem_io_set_hdr(cdev, (struct ulp_mem_io *)skb->head, wr_len, dlen,
+ pm_addr, tid);
+
+ return skb;
+}
+
+static int ddp_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,
+ struct cxgbi_task_tag_info *ttinfo,
+ unsigned int idx, unsigned int npods,
+ struct scatterlist **sg_pp,
+ unsigned int *sg_off)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct sk_buff *skb = ddp_ppod_init_idata(cdev, ppm, idx, npods,
+ csk->tid);
+ struct ulp_mem_io *req;
+ struct ulptx_idata *idata;
+ struct cxgbi_pagepod *ppod;
+ int i;
+
+ if (!skb)
+ return -ENOMEM;
+
+ req = (struct ulp_mem_io *)skb->head;
+ idata = (struct ulptx_idata *)(req + 1);
+ ppod = (struct cxgbi_pagepod *)(idata + 1);
+
+ for (i = 0; i < npods; i++, ppod++)
+ cxgbi_ddp_set_one_ppod(ppod, ttinfo, sg_pp, sg_off);
+
+ cxgbi_skcb_set_flag(skb, SKCBF_TX_MEM_WRITE);
+ cxgbi_skcb_set_flag(skb, SKCBF_TX_FLAG_COMPL);
+ set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
+
+ spin_lock_bh(&csk->lock);
+ cxgbi_sock_skb_entail(csk, skb);
+ spin_unlock_bh(&csk->lock);
+
+ return 0;
+}
+
+static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,
+ struct cxgbi_task_tag_info *ttinfo)
+{
+ unsigned int pidx = ttinfo->idx;
+ unsigned int npods = ttinfo->npods;
+ unsigned int i, cnt;
+ int err = 0;
+ struct scatterlist *sg = ttinfo->sgl;
+ unsigned int offset = 0;
+
+ ttinfo->cid = csk->port_id;
+
+ for (i = 0; i < npods; i += cnt, pidx += cnt) {
+ cnt = npods - i;
+
+ if (cnt > ULPMEM_IDATA_MAX_NPPODS)
+ cnt = ULPMEM_IDATA_MAX_NPPODS;
+ err = ddp_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt,
+ &sg, &offset);
+ if (err < 0)
+ break;
+ }
+
+ return err;
+}
+
+static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid,
+ int pg_idx)
+{
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+
+ if (!pg_idx || pg_idx >= DDP_PGIDX_MAX)
+ return 0;
+
+ skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL);
+ if (!skb)
+ return -ENOMEM;
+
+ /* set up ulp page size */
+ req = (struct cpl_set_tcb_field *)skb->head;
+ INIT_TP_WR(req, csk->tid);
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));
+ req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(0);
+ req->mask = cpu_to_be64(0x3 << 8);
+ req->val = cpu_to_be64(pg_idx << 8);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk, csk->tid, pg_idx);
+
+ reinit_completion(&csk->cmpl);
+ cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
+ wait_for_completion(&csk->cmpl);
+
+ return csk->err;
+}
+
+static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
+ int hcrc, int dcrc)
+{
+ struct sk_buff *skb;
+ struct cpl_set_tcb_field *req;
+
+ if (!hcrc && !dcrc)
+ return 0;
+
+ skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL);
+ if (!skb)
+ return -ENOMEM;
+
+ csk->hcrc_len = (hcrc ? 4 : 0);
+ csk->dcrc_len = (dcrc ? 4 : 0);
+ /* set up ulp submode */
+ req = (struct cpl_set_tcb_field *)skb->head;
+ INIT_TP_WR(req, tid);
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
+ req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid));
+ req->word_cookie = htons(0);
+ req->mask = cpu_to_be64(0x3 << 4);
+ req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) |
+ (dcrc ? ULP_CRC_DATA : 0)) << 4);
+ set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
+
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk, csk->tid, hcrc, dcrc);
+
+ reinit_completion(&csk->cmpl);
+ cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
+ wait_for_completion(&csk->cmpl);
+
+ return csk->err;
+}
+
+static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev)
+{
+ return (struct cxgbi_ppm *)(*((struct cxgb4_lld_info *)
+ (cxgbi_cdev_priv(cdev)))->iscsi_ppm);
+}
+
+static int cxgb4i_ddp_init(struct cxgbi_device *cdev)
+{
+ struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
+ struct net_device *ndev = cdev->ports[0];
+ struct cxgbi_tag_format tformat;
+ int i, err;
+
+ if (!lldi->vr->iscsi.size) {
+ pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name);
+ return -EACCES;
+ }
+
+ cdev->flags |= CXGBI_FLAG_USE_PPOD_OFLDQ;
+
+ memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
+ for (i = 0; i < 4; i++)
+ tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3))
+ & 0xF;
+ cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat);
+
+ pr_info("iscsi_edram.start 0x%x iscsi_edram.size 0x%x",
+ lldi->vr->ppod_edram.start, lldi->vr->ppod_edram.size);
+
+ err = cxgbi_ddp_ppm_setup(lldi->iscsi_ppm, cdev, &tformat,
+ lldi->vr->iscsi.size, lldi->iscsi_llimit,
+ lldi->vr->iscsi.start, 2,
+ lldi->vr->ppod_edram.start,
+ lldi->vr->ppod_edram.size);
+
+ if (err < 0)
+ return err;
+
+ cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
+ cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
+ cdev->csk_ddp_set_map = ddp_set_map;
+ cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
+ lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN);
+ cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
+ lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN);
+ cdev->cdev2ppm = cdev2ppm;
+
+ return 0;
+}
+
+static bool is_memfree(struct adapter *adap)
+{
+ u32 io;
+
+ io = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A);
+ if (is_t5(adap->params.chip)) {
+ if ((io & EXT_MEM0_ENABLE_F) || (io & EXT_MEM1_ENABLE_F))
+ return false;
+ } else if (io & EXT_MEM_ENABLE_F) {
+ return false;
+ }
+
+ return true;
+}
+
+static void *t4_uld_add(const struct cxgb4_lld_info *lldi)
+{
+ struct cxgbi_device *cdev;
+ struct port_info *pi;
+ struct net_device *ndev;
+ struct adapter *adap;
+ struct tid_info *t;
+ u32 max_cmds = CXGB4I_SCSI_HOST_QDEPTH;
+ u32 max_conn = CXGBI_MAX_CONN;
+ int i, rc;
+
+ cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports);
+ if (!cdev) {
+ pr_info("t4 device 0x%p, register failed.\n", lldi);
+ return NULL;
+ }
+ pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n",
+ cdev, lldi->adapter_type, lldi->nports,
+ lldi->ports[0]->name, lldi->nchan, lldi->ntxq,
+ lldi->nrxq, lldi->wr_cred);
+ for (i = 0; i < lldi->nrxq; i++)
+ log_debug(1 << CXGBI_DBG_DEV,
+ "t4 0x%p, rxq id #%d: %u.\n",
+ cdev, i, lldi->rxq_ids[i]);
+
+ memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi));
+ cdev->flags = CXGBI_FLAG_DEV_T4;
+ cdev->pdev = lldi->pdev;
+ cdev->ports = lldi->ports;
+ cdev->nports = lldi->nports;
+ cdev->mtus = lldi->mtus;
+ cdev->nmtus = NMTUS;
+ cdev->rx_credit_thres = (CHELSIO_CHIP_VERSION(lldi->adapter_type) <=
+ CHELSIO_T5) ? cxgb4i_rx_credit_thres : 0;
+ cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN;
+ cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr);
+ cdev->itp = &cxgb4i_iscsi_transport;
+ cdev->owner = THIS_MODULE;
+
+ cdev->pfvf = FW_PFVF_CMD_PFN_V(lldi->pf);
+ pr_info("cdev 0x%p,%s, pfvf %u.\n",
+ cdev, lldi->ports[0]->name, cdev->pfvf);
+
+ rc = cxgb4i_ddp_init(cdev);
+ if (rc) {
+ pr_info("t4 0x%p ddp init failed %d.\n", cdev, rc);
+ goto err_out;
+ }
+
+ ndev = cdev->ports[0];
+ adap = netdev2adap(ndev);
+ if (adap) {
+ t = &adap->tids;
+ if (t->ntids <= CXGBI_MAX_CONN)
+ max_conn = t->ntids;
+
+ if (is_memfree(adap)) {
+ cdev->flags |= CXGBI_FLAG_DEV_ISO_OFF;
+ max_cmds = CXGB4I_SCSI_HOST_QDEPTH >> 2;
+
+ pr_info("%s: 0x%p, tid %u, SO adapter.\n",
+ ndev->name, cdev, t->ntids);
+ }
+ } else {
+ pr_info("%s, 0x%p, NO adapter struct.\n", ndev->name, cdev);
+ }
+
+ /* ISO is enabled in T5/T6 firmware version >= 1.13.43.0 */
+ if (!is_t4(lldi->adapter_type) &&
+ (lldi->fw_vers >= 0x10d2b00) &&
+ !(cdev->flags & CXGBI_FLAG_DEV_ISO_OFF))
+ cdev->skb_iso_txhdr = sizeof(struct cpl_tx_data_iso);
+
+ rc = cxgb4i_ofld_init(cdev);
+ if (rc) {
+ pr_info("t4 0x%p ofld init failed.\n", cdev);
+ goto err_out;
+ }
+
+ cxgb4i_host_template.can_queue = max_cmds;
+ rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, max_conn,
+ &cxgb4i_host_template, cxgb4i_stt);
+ if (rc)
+ goto err_out;
+
+ for (i = 0; i < cdev->nports; i++) {
+ pi = netdev_priv(lldi->ports[i]);
+ cdev->hbas[i]->port_id = pi->port_id;
+ }
+ return cdev;
+
+err_out:
+ cxgbi_device_unregister(cdev);
+ return ERR_PTR(-ENOMEM);
+}
+
+#define RX_PULL_LEN 128
+static int t4_uld_rx_handler(void *handle, const __be64 *rsp,
+ const struct pkt_gl *pgl)
+{
+ const struct cpl_act_establish *rpl;
+ struct sk_buff *skb;
+ unsigned int opc;
+ struct cxgbi_device *cdev = handle;
+
+ if (pgl == NULL) {
+ unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8;
+
+ skb = alloc_wr(len, 0, GFP_ATOMIC);
+ if (!skb)
+ goto nomem;
+ skb_copy_to_linear_data(skb, &rsp[1], len);
+ } else {
+ if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) {
+ pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
+ pgl->va, be64_to_cpu(*rsp),
+ be64_to_cpu(*(u64 *)pgl->va),
+ pgl->tot_len);
+ return 0;
+ }
+ skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN);
+ if (unlikely(!skb))
+ goto nomem;
+ }
+
+ rpl = (struct cpl_act_establish *)skb->data;
+ opc = rpl->ot.opcode;
+ log_debug(1 << CXGBI_DBG_TOE,
+ "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
+ cdev, opc, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb);
+ if (opc >= ARRAY_SIZE(cxgb4i_cplhandlers) || !cxgb4i_cplhandlers[opc]) {
+ pr_err("No handler for opcode 0x%x.\n", opc);
+ __kfree_skb(skb);
+ } else
+ cxgb4i_cplhandlers[opc](cdev, skb);
+
+ return 0;
+nomem:
+ log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n");
+ return 1;
+}
+
+static int t4_uld_state_change(void *handle, enum cxgb4_state state)
+{
+ struct cxgbi_device *cdev = handle;
+
+ switch (state) {
+ case CXGB4_STATE_UP:
+ pr_info("cdev 0x%p, UP.\n", cdev);
+ break;
+ case CXGB4_STATE_START_RECOVERY:
+ pr_info("cdev 0x%p, RECOVERY.\n", cdev);
+ /* close all connections */
+ break;
+ case CXGB4_STATE_DOWN:
+ pr_info("cdev 0x%p, DOWN.\n", cdev);
+ break;
+ case CXGB4_STATE_DETACH:
+ pr_info("cdev 0x%p, DETACH.\n", cdev);
+ cxgbi_device_unregister(cdev);
+ break;
+ default:
+ pr_info("cdev 0x%p, unknown state %d.\n", cdev, state);
+ break;
+ }
+ return 0;
+}
+
+#ifdef CONFIG_CHELSIO_T4_DCB
+static int
+cxgb4_dcb_change_notify(struct notifier_block *self, unsigned long val,
+ void *data)
+{
+ int i, port = 0xFF;
+ struct net_device *ndev;
+ struct cxgbi_device *cdev = NULL;
+ struct dcb_app_type *iscsi_app = data;
+ struct cxgbi_ports_map *pmap;
+ u8 priority;
+
+ if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_IEEE) {
+ if ((iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_STREAM) &&
+ (iscsi_app->app.selector != IEEE_8021QAZ_APP_SEL_ANY))
+ return NOTIFY_DONE;
+
+ priority = iscsi_app->app.priority;
+ } else if (iscsi_app->dcbx & DCB_CAP_DCBX_VER_CEE) {
+ if (iscsi_app->app.selector != DCB_APP_IDTYPE_PORTNUM)
+ return NOTIFY_DONE;
+
+ if (!iscsi_app->app.priority)
+ return NOTIFY_DONE;
+
+ priority = ffs(iscsi_app->app.priority) - 1;
+ } else {
+ return NOTIFY_DONE;
+ }
+
+ if (iscsi_app->app.protocol != 3260)
+ return NOTIFY_DONE;
+
+ log_debug(1 << CXGBI_DBG_ISCSI, "iSCSI priority for ifid %d is %u\n",
+ iscsi_app->ifindex, priority);
+
+ ndev = dev_get_by_index(&init_net, iscsi_app->ifindex);
+ if (!ndev)
+ return NOTIFY_DONE;
+
+ cdev = cxgbi_device_find_by_netdev_rcu(ndev, &port);
+
+ dev_put(ndev);
+ if (!cdev)
+ return NOTIFY_DONE;
+
+ pmap = &cdev->pmap;
+
+ for (i = 0; i < pmap->used; i++) {
+ if (pmap->port_csk[i]) {
+ struct cxgbi_sock *csk = pmap->port_csk[i];
+
+ if (csk->dcb_priority != priority) {
+ iscsi_conn_failure(csk->user_data,
+ ISCSI_ERR_CONN_FAILED);
+ pr_info("Restarting iSCSI connection %p with "
+ "priority %u->%u.\n", csk,
+ csk->dcb_priority, priority);
+ }
+ }
+ }
+ return NOTIFY_OK;
+}
+#endif
+
+static int __init cxgb4i_init_module(void)
+{
+ int rc;
+
+ printk(KERN_INFO "%s", version);
+
+ rc = cxgbi_iscsi_init(&cxgb4i_iscsi_transport, &cxgb4i_stt);
+ if (rc < 0)
+ return rc;
+ cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info);
+
+#ifdef CONFIG_CHELSIO_T4_DCB
+ pr_info("%s dcb enabled.\n", DRV_MODULE_NAME);
+ register_dcbevent_notifier(&cxgb4_dcb_change);
+#endif
+ return 0;
+}
+
+static void __exit cxgb4i_exit_module(void)
+{
+#ifdef CONFIG_CHELSIO_T4_DCB
+ unregister_dcbevent_notifier(&cxgb4_dcb_change);
+#endif
+ cxgb4_unregister_uld(CXGB4_ULD_ISCSI);
+ cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4);
+ cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport, &cxgb4i_stt);
+}
+
+module_init(cxgb4i_init_module);
+module_exit(cxgb4i_exit_module);
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.h b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.h
new file mode 100644
index 000000000..2fd9c76fc
--- /dev/null
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.h
@@ -0,0 +1,28 @@
+/*
+ * cxgb4i.h: Chelsio T4 iSCSI driver.
+ *
+ * Copyright (c) 2010-2015 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * Written by: Karen Xie (kxie@chelsio.com)
+ * Written by: Rakesh Ranjan (rranjan@chelsio.com)
+ */
+
+#ifndef __CXGB4I_H__
+#define __CXGB4I_H__
+
+#define CXGB4I_SCSI_HOST_QDEPTH 1024
+#define CXGB4I_MAX_CONN 16384
+#define CXGB4I_MAX_TARGET CXGB4I_MAX_CONN
+#define CXGB4I_MAX_LUN 0x1000
+
+/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */
+#define CXGB4I_TX_HEADER_LEN \
+ (sizeof(struct fw_ofld_tx_data_wr) + sizeof(struct sge_opaque_hdr))
+
+#define T5_ISS_VALID (1 << 18)
+
+#endif /* __CXGB4I_H__ */
diff --git a/drivers/scsi/cxgbi/libcxgbi.c b/drivers/scsi/cxgbi/libcxgbi.c
new file mode 100644
index 000000000..506b56167
--- /dev/null
+++ b/drivers/scsi/cxgbi/libcxgbi.c
@@ -0,0 +1,3098 @@
+/*
+ * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver.
+ *
+ * Copyright (c) 2010-2015 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * Written by: Karen Xie (kxie@chelsio.com)
+ * Written by: Rakesh Ranjan (rranjan@chelsio.com)
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/skbuff.h>
+#include <linux/crypto.h>
+#include <linux/scatterlist.h>
+#include <linux/pci.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_host.h>
+#include <linux/if_vlan.h>
+#include <linux/inet.h>
+#include <net/dst.h>
+#include <net/route.h>
+#include <net/ipv6.h>
+#include <net/ip6_route.h>
+#include <net/addrconf.h>
+
+#include <linux/inetdevice.h> /* ip_dev_find */
+#include <linux/module.h>
+#include <net/tcp.h>
+
+static unsigned int dbg_level;
+
+#include "libcxgbi.h"
+
+#define DRV_MODULE_NAME "libcxgbi"
+#define DRV_MODULE_DESC "Chelsio iSCSI driver library"
+#define DRV_MODULE_VERSION "0.9.1-ko"
+#define DRV_MODULE_RELDATE "Apr. 2015"
+
+static char version[] =
+ DRV_MODULE_DESC " " DRV_MODULE_NAME
+ " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
+
+MODULE_AUTHOR("Chelsio Communications, Inc.");
+MODULE_DESCRIPTION(DRV_MODULE_DESC);
+MODULE_VERSION(DRV_MODULE_VERSION);
+MODULE_LICENSE("GPL");
+
+module_param(dbg_level, uint, 0644);
+MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)");
+
+
+/*
+ * cxgbi device management
+ * maintains a list of the cxgbi devices
+ */
+static LIST_HEAD(cdev_list);
+static DEFINE_MUTEX(cdev_mutex);
+
+static LIST_HEAD(cdev_rcu_list);
+static DEFINE_SPINLOCK(cdev_rcu_lock);
+
+static inline void cxgbi_decode_sw_tag(u32 sw_tag, int *idx, int *age)
+{
+ if (age)
+ *age = sw_tag & 0x7FFF;
+ if (idx)
+ *idx = (sw_tag >> 16) & 0x7FFF;
+}
+
+int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base,
+ unsigned int max_conn)
+{
+ struct cxgbi_ports_map *pmap = &cdev->pmap;
+
+ pmap->port_csk = kvzalloc(array_size(max_conn,
+ sizeof(struct cxgbi_sock *)),
+ GFP_KERNEL | __GFP_NOWARN);
+ if (!pmap->port_csk) {
+ pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn);
+ return -ENOMEM;
+ }
+
+ pmap->max_connect = max_conn;
+ pmap->sport_base = base;
+ spin_lock_init(&pmap->lock);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create);
+
+void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev)
+{
+ struct cxgbi_ports_map *pmap = &cdev->pmap;
+ struct cxgbi_sock *csk;
+ int i;
+
+ for (i = 0; i < pmap->max_connect; i++) {
+ if (pmap->port_csk[i]) {
+ csk = pmap->port_csk[i];
+ pmap->port_csk[i] = NULL;
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "csk 0x%p, cdev 0x%p, offload down.\n",
+ csk, cdev);
+ spin_lock_bh(&csk->lock);
+ cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN);
+ cxgbi_sock_closed(csk);
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+ }
+ }
+}
+EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup);
+
+static inline void cxgbi_device_destroy(struct cxgbi_device *cdev)
+{
+ log_debug(1 << CXGBI_DBG_DEV,
+ "cdev 0x%p, p# %u.\n", cdev, cdev->nports);
+ cxgbi_hbas_remove(cdev);
+ cxgbi_device_portmap_cleanup(cdev);
+ if (cdev->cdev2ppm)
+ cxgbi_ppm_release(cdev->cdev2ppm(cdev));
+ if (cdev->pmap.max_connect)
+ kvfree(cdev->pmap.port_csk);
+ kfree(cdev);
+}
+
+struct cxgbi_device *cxgbi_device_register(unsigned int extra,
+ unsigned int nports)
+{
+ struct cxgbi_device *cdev;
+
+ cdev = kzalloc(sizeof(*cdev) + extra + nports *
+ (sizeof(struct cxgbi_hba *) +
+ sizeof(struct net_device *)),
+ GFP_KERNEL);
+ if (!cdev) {
+ pr_warn("nport %d, OOM.\n", nports);
+ return NULL;
+ }
+ cdev->ports = (struct net_device **)(cdev + 1);
+ cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports *
+ sizeof(struct net_device *));
+ if (extra)
+ cdev->dd_data = ((char *)cdev->hbas) +
+ nports * sizeof(struct cxgbi_hba *);
+ spin_lock_init(&cdev->pmap.lock);
+
+ mutex_lock(&cdev_mutex);
+ list_add_tail(&cdev->list_head, &cdev_list);
+ mutex_unlock(&cdev_mutex);
+
+ spin_lock(&cdev_rcu_lock);
+ list_add_tail_rcu(&cdev->rcu_node, &cdev_rcu_list);
+ spin_unlock(&cdev_rcu_lock);
+
+ log_debug(1 << CXGBI_DBG_DEV,
+ "cdev 0x%p, p# %u.\n", cdev, nports);
+ return cdev;
+}
+EXPORT_SYMBOL_GPL(cxgbi_device_register);
+
+void cxgbi_device_unregister(struct cxgbi_device *cdev)
+{
+ log_debug(1 << CXGBI_DBG_DEV,
+ "cdev 0x%p, p# %u,%s.\n",
+ cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : "");
+
+ mutex_lock(&cdev_mutex);
+ list_del(&cdev->list_head);
+ mutex_unlock(&cdev_mutex);
+
+ spin_lock(&cdev_rcu_lock);
+ list_del_rcu(&cdev->rcu_node);
+ spin_unlock(&cdev_rcu_lock);
+ synchronize_rcu();
+
+ cxgbi_device_destroy(cdev);
+}
+EXPORT_SYMBOL_GPL(cxgbi_device_unregister);
+
+void cxgbi_device_unregister_all(unsigned int flag)
+{
+ struct cxgbi_device *cdev, *tmp;
+
+ mutex_lock(&cdev_mutex);
+ list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
+ if ((cdev->flags & flag) == flag) {
+ mutex_unlock(&cdev_mutex);
+ cxgbi_device_unregister(cdev);
+ mutex_lock(&cdev_mutex);
+ }
+ }
+ mutex_unlock(&cdev_mutex);
+}
+EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all);
+
+struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev)
+{
+ struct cxgbi_device *cdev, *tmp;
+
+ mutex_lock(&cdev_mutex);
+ list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
+ if (cdev->lldev == lldev) {
+ mutex_unlock(&cdev_mutex);
+ return cdev;
+ }
+ }
+ mutex_unlock(&cdev_mutex);
+
+ log_debug(1 << CXGBI_DBG_DEV,
+ "lldev 0x%p, NO match found.\n", lldev);
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev);
+
+struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev,
+ int *port)
+{
+ struct net_device *vdev = NULL;
+ struct cxgbi_device *cdev, *tmp;
+ int i;
+
+ if (is_vlan_dev(ndev)) {
+ vdev = ndev;
+ ndev = vlan_dev_real_dev(ndev);
+ log_debug(1 << CXGBI_DBG_DEV,
+ "vlan dev %s -> %s.\n", vdev->name, ndev->name);
+ }
+
+ mutex_lock(&cdev_mutex);
+ list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
+ for (i = 0; i < cdev->nports; i++) {
+ if (ndev == cdev->ports[i]) {
+ cdev->hbas[i]->vdev = vdev;
+ mutex_unlock(&cdev_mutex);
+ if (port)
+ *port = i;
+ return cdev;
+ }
+ }
+ }
+ mutex_unlock(&cdev_mutex);
+ log_debug(1 << CXGBI_DBG_DEV,
+ "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev);
+
+struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev,
+ int *port)
+{
+ struct net_device *vdev = NULL;
+ struct cxgbi_device *cdev;
+ int i;
+
+ if (is_vlan_dev(ndev)) {
+ vdev = ndev;
+ ndev = vlan_dev_real_dev(ndev);
+ pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
+ }
+
+ rcu_read_lock();
+ list_for_each_entry_rcu(cdev, &cdev_rcu_list, rcu_node) {
+ for (i = 0; i < cdev->nports; i++) {
+ if (ndev == cdev->ports[i]) {
+ cdev->hbas[i]->vdev = vdev;
+ rcu_read_unlock();
+ if (port)
+ *port = i;
+ return cdev;
+ }
+ }
+ }
+ rcu_read_unlock();
+
+ log_debug(1 << CXGBI_DBG_DEV,
+ "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu);
+
+static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev,
+ int *port)
+{
+ struct net_device *vdev = NULL;
+ struct cxgbi_device *cdev, *tmp;
+ int i;
+
+ if (is_vlan_dev(ndev)) {
+ vdev = ndev;
+ ndev = vlan_dev_real_dev(ndev);
+ pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
+ }
+
+ mutex_lock(&cdev_mutex);
+ list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
+ for (i = 0; i < cdev->nports; i++) {
+ if (!memcmp(ndev->dev_addr, cdev->ports[i]->dev_addr,
+ MAX_ADDR_LEN)) {
+ cdev->hbas[i]->vdev = vdev;
+ mutex_unlock(&cdev_mutex);
+ if (port)
+ *port = i;
+ return cdev;
+ }
+ }
+ }
+ mutex_unlock(&cdev_mutex);
+ log_debug(1 << CXGBI_DBG_DEV,
+ "ndev 0x%p, %s, NO match mac found.\n",
+ ndev, ndev->name);
+ return NULL;
+}
+
+void cxgbi_hbas_remove(struct cxgbi_device *cdev)
+{
+ int i;
+ struct cxgbi_hba *chba;
+
+ log_debug(1 << CXGBI_DBG_DEV,
+ "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
+
+ for (i = 0; i < cdev->nports; i++) {
+ chba = cdev->hbas[i];
+ if (chba) {
+ cdev->hbas[i] = NULL;
+ iscsi_host_remove(chba->shost);
+ pci_dev_put(cdev->pdev);
+ iscsi_host_free(chba->shost);
+ }
+ }
+}
+EXPORT_SYMBOL_GPL(cxgbi_hbas_remove);
+
+int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun,
+ unsigned int max_conns, struct scsi_host_template *sht,
+ struct scsi_transport_template *stt)
+{
+ struct cxgbi_hba *chba;
+ struct Scsi_Host *shost;
+ int i, err;
+
+ log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
+
+ for (i = 0; i < cdev->nports; i++) {
+ shost = iscsi_host_alloc(sht, sizeof(*chba), 1);
+ if (!shost) {
+ pr_info("0x%p, p%d, %s, host alloc failed.\n",
+ cdev, i, cdev->ports[i]->name);
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ shost->transportt = stt;
+ shost->max_lun = max_lun;
+ shost->max_id = max_conns - 1;
+ shost->max_channel = 0;
+ shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
+
+ chba = iscsi_host_priv(shost);
+ chba->cdev = cdev;
+ chba->ndev = cdev->ports[i];
+ chba->shost = shost;
+
+ shost->can_queue = sht->can_queue - ISCSI_MGMT_CMDS_MAX;
+
+ log_debug(1 << CXGBI_DBG_DEV,
+ "cdev 0x%p, p#%d %s: chba 0x%p.\n",
+ cdev, i, cdev->ports[i]->name, chba);
+
+ pci_dev_get(cdev->pdev);
+ err = iscsi_host_add(shost, &cdev->pdev->dev);
+ if (err) {
+ pr_info("cdev 0x%p, p#%d %s, host add failed.\n",
+ cdev, i, cdev->ports[i]->name);
+ pci_dev_put(cdev->pdev);
+ scsi_host_put(shost);
+ goto err_out;
+ }
+
+ cdev->hbas[i] = chba;
+ }
+
+ return 0;
+
+err_out:
+ cxgbi_hbas_remove(cdev);
+ return err;
+}
+EXPORT_SYMBOL_GPL(cxgbi_hbas_add);
+
+/*
+ * iSCSI offload
+ *
+ * - source port management
+ * To find a free source port in the port allocation map we use a very simple
+ * rotor scheme to look for the next free port.
+ *
+ * If a source port has been specified make sure that it doesn't collide with
+ * our normal source port allocation map. If it's outside the range of our
+ * allocation/deallocation scheme just let them use it.
+ *
+ * If the source port is outside our allocation range, the caller is
+ * responsible for keeping track of their port usage.
+ */
+
+static struct cxgbi_sock *find_sock_on_port(struct cxgbi_device *cdev,
+ unsigned char port_id)
+{
+ struct cxgbi_ports_map *pmap = &cdev->pmap;
+ unsigned int i;
+ unsigned int used;
+
+ if (!pmap->max_connect || !pmap->used)
+ return NULL;
+
+ spin_lock_bh(&pmap->lock);
+ used = pmap->used;
+ for (i = 0; used && i < pmap->max_connect; i++) {
+ struct cxgbi_sock *csk = pmap->port_csk[i];
+
+ if (csk) {
+ if (csk->port_id == port_id) {
+ spin_unlock_bh(&pmap->lock);
+ return csk;
+ }
+ used--;
+ }
+ }
+ spin_unlock_bh(&pmap->lock);
+
+ return NULL;
+}
+
+static int sock_get_port(struct cxgbi_sock *csk)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct cxgbi_ports_map *pmap = &cdev->pmap;
+ unsigned int start;
+ int idx;
+ __be16 *port;
+
+ if (!pmap->max_connect) {
+ pr_err("cdev 0x%p, p#%u %s, NO port map.\n",
+ cdev, csk->port_id, cdev->ports[csk->port_id]->name);
+ return -EADDRNOTAVAIL;
+ }
+
+ if (csk->csk_family == AF_INET)
+ port = &csk->saddr.sin_port;
+ else /* ipv6 */
+ port = &csk->saddr6.sin6_port;
+
+ if (*port) {
+ pr_err("source port NON-ZERO %u.\n",
+ ntohs(*port));
+ return -EADDRINUSE;
+ }
+
+ spin_lock_bh(&pmap->lock);
+ if (pmap->used >= pmap->max_connect) {
+ spin_unlock_bh(&pmap->lock);
+ pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n",
+ cdev, csk->port_id, cdev->ports[csk->port_id]->name);
+ return -EADDRNOTAVAIL;
+ }
+
+ start = idx = pmap->next;
+ do {
+ if (++idx >= pmap->max_connect)
+ idx = 0;
+ if (!pmap->port_csk[idx]) {
+ pmap->used++;
+ *port = htons(pmap->sport_base + idx);
+ pmap->next = idx;
+ pmap->port_csk[idx] = csk;
+ spin_unlock_bh(&pmap->lock);
+ cxgbi_sock_get(csk);
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "cdev 0x%p, p#%u %s, p %u, %u.\n",
+ cdev, csk->port_id,
+ cdev->ports[csk->port_id]->name,
+ pmap->sport_base + idx, pmap->next);
+ return 0;
+ }
+ } while (idx != start);
+ spin_unlock_bh(&pmap->lock);
+
+ /* should not happen */
+ pr_warn("cdev 0x%p, p#%u %s, next %u?\n",
+ cdev, csk->port_id, cdev->ports[csk->port_id]->name,
+ pmap->next);
+ return -EADDRNOTAVAIL;
+}
+
+static void sock_put_port(struct cxgbi_sock *csk)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct cxgbi_ports_map *pmap = &cdev->pmap;
+ __be16 *port;
+
+ if (csk->csk_family == AF_INET)
+ port = &csk->saddr.sin_port;
+ else /* ipv6 */
+ port = &csk->saddr6.sin6_port;
+
+ if (*port) {
+ int idx = ntohs(*port) - pmap->sport_base;
+
+ *port = 0;
+ if (idx < 0 || idx >= pmap->max_connect) {
+ pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n",
+ cdev, csk->port_id,
+ cdev->ports[csk->port_id]->name,
+ ntohs(*port));
+ return;
+ }
+
+ spin_lock_bh(&pmap->lock);
+ pmap->port_csk[idx] = NULL;
+ pmap->used--;
+ spin_unlock_bh(&pmap->lock);
+
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "cdev 0x%p, p#%u %s, release %u.\n",
+ cdev, csk->port_id, cdev->ports[csk->port_id]->name,
+ pmap->sport_base + idx);
+
+ cxgbi_sock_put(csk);
+ }
+}
+
+/*
+ * iscsi tcp connection
+ */
+void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk)
+{
+ if (csk->cpl_close) {
+ kfree_skb(csk->cpl_close);
+ csk->cpl_close = NULL;
+ }
+ if (csk->cpl_abort_req) {
+ kfree_skb(csk->cpl_abort_req);
+ csk->cpl_abort_req = NULL;
+ }
+ if (csk->cpl_abort_rpl) {
+ kfree_skb(csk->cpl_abort_rpl);
+ csk->cpl_abort_rpl = NULL;
+ }
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs);
+
+static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev)
+{
+ struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO);
+
+ if (!csk) {
+ pr_info("alloc csk %zu failed.\n", sizeof(*csk));
+ return NULL;
+ }
+
+ if (cdev->csk_alloc_cpls(csk) < 0) {
+ pr_info("csk 0x%p, alloc cpls failed.\n", csk);
+ kfree(csk);
+ return NULL;
+ }
+
+ spin_lock_init(&csk->lock);
+ kref_init(&csk->refcnt);
+ skb_queue_head_init(&csk->receive_queue);
+ skb_queue_head_init(&csk->write_queue);
+ timer_setup(&csk->retry_timer, NULL, 0);
+ init_completion(&csk->cmpl);
+ rwlock_init(&csk->callback_lock);
+ csk->cdev = cdev;
+ csk->flags = 0;
+ cxgbi_sock_set_state(csk, CTP_CLOSED);
+
+ log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk);
+
+ return csk;
+}
+
+static struct rtable *find_route_ipv4(struct flowi4 *fl4,
+ __be32 saddr, __be32 daddr,
+ __be16 sport, __be16 dport, u8 tos,
+ int ifindex)
+{
+ struct rtable *rt;
+
+ rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr,
+ dport, sport, IPPROTO_TCP, tos, ifindex);
+ if (IS_ERR(rt))
+ return NULL;
+
+ return rt;
+}
+
+static struct cxgbi_sock *
+cxgbi_check_route(struct sockaddr *dst_addr, int ifindex)
+{
+ struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr;
+ struct dst_entry *dst;
+ struct net_device *ndev;
+ struct cxgbi_device *cdev;
+ struct rtable *rt = NULL;
+ struct neighbour *n;
+ struct flowi4 fl4;
+ struct cxgbi_sock *csk = NULL;
+ unsigned int mtu = 0;
+ int port = 0xFFFF;
+ int err = 0;
+
+ rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0,
+ daddr->sin_port, 0, ifindex);
+ if (!rt) {
+ pr_info("no route to ipv4 0x%x, port %u.\n",
+ be32_to_cpu(daddr->sin_addr.s_addr),
+ be16_to_cpu(daddr->sin_port));
+ err = -ENETUNREACH;
+ goto err_out;
+ }
+ dst = &rt->dst;
+ n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr);
+ if (!n) {
+ err = -ENODEV;
+ goto rel_rt;
+ }
+ ndev = n->dev;
+
+ if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
+ pr_info("multi-cast route %pI4, port %u, dev %s.\n",
+ &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
+ ndev->name);
+ err = -ENETUNREACH;
+ goto rel_neigh;
+ }
+
+ if (ndev->flags & IFF_LOOPBACK) {
+ ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr);
+ if (!ndev) {
+ err = -ENETUNREACH;
+ goto rel_neigh;
+ }
+ mtu = ndev->mtu;
+ pr_info("rt dev %s, loopback -> %s, mtu %u.\n",
+ n->dev->name, ndev->name, mtu);
+ }
+
+ if (!(ndev->flags & IFF_UP) || !netif_carrier_ok(ndev)) {
+ pr_info("%s interface not up.\n", ndev->name);
+ err = -ENETDOWN;
+ goto rel_neigh;
+ }
+
+ cdev = cxgbi_device_find_by_netdev(ndev, &port);
+ if (!cdev)
+ cdev = cxgbi_device_find_by_mac(ndev, &port);
+ if (!cdev) {
+ pr_info("dst %pI4, %s, NOT cxgbi device.\n",
+ &daddr->sin_addr.s_addr, ndev->name);
+ err = -ENETUNREACH;
+ goto rel_neigh;
+ }
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n",
+ &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
+ port, ndev->name, cdev);
+
+ csk = cxgbi_sock_create(cdev);
+ if (!csk) {
+ err = -ENOMEM;
+ goto rel_neigh;
+ }
+ csk->cdev = cdev;
+ csk->port_id = port;
+ csk->mtu = mtu;
+ csk->dst = dst;
+
+ csk->csk_family = AF_INET;
+ csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr;
+ csk->daddr.sin_port = daddr->sin_port;
+ csk->daddr.sin_family = daddr->sin_family;
+ csk->saddr.sin_family = daddr->sin_family;
+ csk->saddr.sin_addr.s_addr = fl4.saddr;
+ neigh_release(n);
+
+ return csk;
+
+rel_neigh:
+ neigh_release(n);
+
+rel_rt:
+ ip_rt_put(rt);
+err_out:
+ return ERR_PTR(err);
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr,
+ const struct in6_addr *daddr,
+ int ifindex)
+{
+ struct flowi6 fl;
+
+ memset(&fl, 0, sizeof(fl));
+ fl.flowi6_oif = ifindex;
+ if (saddr)
+ memcpy(&fl.saddr, saddr, sizeof(struct in6_addr));
+ if (daddr)
+ memcpy(&fl.daddr, daddr, sizeof(struct in6_addr));
+ return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl);
+}
+
+static struct cxgbi_sock *
+cxgbi_check_route6(struct sockaddr *dst_addr, int ifindex)
+{
+ struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr;
+ struct dst_entry *dst;
+ struct net_device *ndev;
+ struct cxgbi_device *cdev;
+ struct rt6_info *rt = NULL;
+ struct neighbour *n;
+ struct in6_addr pref_saddr;
+ struct cxgbi_sock *csk = NULL;
+ unsigned int mtu = 0;
+ int port = 0xFFFF;
+ int err = 0;
+
+ rt = find_route_ipv6(NULL, &daddr6->sin6_addr, ifindex);
+
+ if (!rt) {
+ pr_info("no route to ipv6 %pI6 port %u\n",
+ daddr6->sin6_addr.s6_addr,
+ be16_to_cpu(daddr6->sin6_port));
+ err = -ENETUNREACH;
+ goto err_out;
+ }
+
+ dst = &rt->dst;
+
+ n = dst_neigh_lookup(dst, &daddr6->sin6_addr);
+
+ if (!n) {
+ pr_info("%pI6, port %u, dst no neighbour.\n",
+ daddr6->sin6_addr.s6_addr,
+ be16_to_cpu(daddr6->sin6_port));
+ err = -ENETUNREACH;
+ goto rel_rt;
+ }
+ ndev = n->dev;
+
+ if (!(ndev->flags & IFF_UP) || !netif_carrier_ok(ndev)) {
+ pr_info("%s interface not up.\n", ndev->name);
+ err = -ENETDOWN;
+ goto rel_rt;
+ }
+
+ if (ipv6_addr_is_multicast(&daddr6->sin6_addr)) {
+ pr_info("multi-cast route %pI6 port %u, dev %s.\n",
+ daddr6->sin6_addr.s6_addr,
+ ntohs(daddr6->sin6_port), ndev->name);
+ err = -ENETUNREACH;
+ goto rel_rt;
+ }
+
+ cdev = cxgbi_device_find_by_netdev(ndev, &port);
+ if (!cdev)
+ cdev = cxgbi_device_find_by_mac(ndev, &port);
+ if (!cdev) {
+ pr_info("dst %pI6 %s, NOT cxgbi device.\n",
+ daddr6->sin6_addr.s6_addr, ndev->name);
+ err = -ENETUNREACH;
+ goto rel_rt;
+ }
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n",
+ daddr6->sin6_addr.s6_addr, ntohs(daddr6->sin6_port), port,
+ ndev->name, cdev);
+
+ csk = cxgbi_sock_create(cdev);
+ if (!csk) {
+ err = -ENOMEM;
+ goto rel_rt;
+ }
+ csk->cdev = cdev;
+ csk->port_id = port;
+ csk->mtu = mtu;
+ csk->dst = dst;
+
+ rt6_get_prefsrc(rt, &pref_saddr);
+ if (ipv6_addr_any(&pref_saddr)) {
+ struct inet6_dev *idev = ip6_dst_idev((struct dst_entry *)rt);
+
+ err = ipv6_dev_get_saddr(&init_net, idev ? idev->dev : NULL,
+ &daddr6->sin6_addr, 0, &pref_saddr);
+ if (err) {
+ pr_info("failed to get source address to reach %pI6\n",
+ &daddr6->sin6_addr);
+ goto rel_rt;
+ }
+ }
+
+ csk->csk_family = AF_INET6;
+ csk->daddr6.sin6_addr = daddr6->sin6_addr;
+ csk->daddr6.sin6_port = daddr6->sin6_port;
+ csk->daddr6.sin6_family = daddr6->sin6_family;
+ csk->saddr6.sin6_family = daddr6->sin6_family;
+ csk->saddr6.sin6_addr = pref_saddr;
+
+ neigh_release(n);
+ return csk;
+
+rel_rt:
+ if (n)
+ neigh_release(n);
+
+ ip6_rt_put(rt);
+ if (csk)
+ cxgbi_sock_closed(csk);
+err_out:
+ return ERR_PTR(err);
+}
+#endif /* IS_ENABLED(CONFIG_IPV6) */
+
+void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn,
+ unsigned int opt)
+{
+ csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn;
+ dst_confirm(csk->dst);
+ smp_mb();
+ cxgbi_sock_set_state(csk, CTP_ESTABLISHED);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_established);
+
+static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk)
+{
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n",
+ csk, csk->state, csk->flags, csk->user_data);
+
+ if (csk->state != CTP_ESTABLISHED) {
+ read_lock_bh(&csk->callback_lock);
+ if (csk->user_data)
+ iscsi_conn_failure(csk->user_data,
+ ISCSI_ERR_TCP_CONN_CLOSE);
+ read_unlock_bh(&csk->callback_lock);
+ }
+}
+
+void cxgbi_sock_closed(struct cxgbi_sock *csk)
+{
+ log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, (csk)->state, (csk)->flags, (csk)->tid);
+ cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
+ if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED)
+ return;
+ if (csk->saddr.sin_port)
+ sock_put_port(csk);
+ if (csk->dst)
+ dst_release(csk->dst);
+ csk->cdev->csk_release_offload_resources(csk);
+ cxgbi_sock_set_state(csk, CTP_CLOSED);
+ cxgbi_inform_iscsi_conn_closing(csk);
+ cxgbi_sock_put(csk);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_closed);
+
+static void need_active_close(struct cxgbi_sock *csk)
+{
+ int data_lost;
+ int close_req = 0;
+
+ log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, (csk)->state, (csk)->flags, (csk)->tid);
+ spin_lock_bh(&csk->lock);
+ if (csk->dst)
+ dst_confirm(csk->dst);
+ data_lost = skb_queue_len(&csk->receive_queue);
+ __skb_queue_purge(&csk->receive_queue);
+
+ if (csk->state == CTP_ACTIVE_OPEN)
+ cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
+ else if (csk->state == CTP_ESTABLISHED) {
+ close_req = 1;
+ cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE);
+ } else if (csk->state == CTP_PASSIVE_CLOSE) {
+ close_req = 1;
+ cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
+ }
+
+ if (close_req) {
+ if (!cxgbi_sock_flag(csk, CTPF_LOGOUT_RSP_RCVD) ||
+ data_lost)
+ csk->cdev->csk_send_abort_req(csk);
+ else
+ csk->cdev->csk_send_close_req(csk);
+ }
+
+ spin_unlock_bh(&csk->lock);
+}
+
+void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno)
+{
+ pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n",
+ csk, csk->state, csk->flags,
+ &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port,
+ &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port,
+ errno);
+
+ cxgbi_sock_set_state(csk, CTP_CONNECTING);
+ csk->err = errno;
+ cxgbi_sock_closed(csk);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open);
+
+void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb)
+{
+ struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk;
+ struct module *owner = csk->cdev->owner;
+
+ log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, (csk)->state, (csk)->flags, (csk)->tid);
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+ if (csk->state == CTP_ACTIVE_OPEN)
+ cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH);
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+ __kfree_skb(skb);
+
+ module_put(owner);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure);
+
+void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk)
+{
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+
+ cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD);
+ if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
+ cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING);
+ if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD))
+ pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ cxgbi_sock_closed(csk);
+ }
+
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl);
+
+void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk)
+{
+ log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, (csk)->state, (csk)->flags, (csk)->tid);
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+
+ if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
+ goto done;
+
+ switch (csk->state) {
+ case CTP_ESTABLISHED:
+ cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE);
+ break;
+ case CTP_ACTIVE_CLOSE:
+ cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
+ break;
+ case CTP_CLOSE_WAIT_1:
+ cxgbi_sock_closed(csk);
+ break;
+ case CTP_ABORTING:
+ break;
+ default:
+ pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ }
+ cxgbi_inform_iscsi_conn_closing(csk);
+done:
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close);
+
+void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt)
+{
+ log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
+ csk, (csk)->state, (csk)->flags, (csk)->tid);
+ cxgbi_sock_get(csk);
+ spin_lock_bh(&csk->lock);
+
+ csk->snd_una = snd_nxt - 1;
+ if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
+ goto done;
+
+ switch (csk->state) {
+ case CTP_ACTIVE_CLOSE:
+ cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1);
+ break;
+ case CTP_CLOSE_WAIT_1:
+ case CTP_CLOSE_WAIT_2:
+ cxgbi_sock_closed(csk);
+ break;
+ case CTP_ABORTING:
+ break;
+ default:
+ pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ }
+done:
+ spin_unlock_bh(&csk->lock);
+ cxgbi_sock_put(csk);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl);
+
+void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits,
+ unsigned int snd_una, int seq_chk)
+{
+ log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n",
+ csk, csk->state, csk->flags, csk->tid, credits,
+ csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk);
+
+ spin_lock_bh(&csk->lock);
+
+ csk->wr_cred += credits;
+ if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred)
+ csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred;
+
+ while (credits) {
+ struct sk_buff *p = cxgbi_sock_peek_wr(csk);
+
+ if (unlikely(!p)) {
+ pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n",
+ csk, csk->state, csk->flags, csk->tid, credits,
+ csk->wr_cred, csk->wr_una_cred);
+ break;
+ }
+
+ if (unlikely(credits < p->csum)) {
+ pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n",
+ csk, csk->state, csk->flags, csk->tid,
+ credits, csk->wr_cred, csk->wr_una_cred,
+ p->csum);
+ p->csum -= credits;
+ break;
+ } else {
+ cxgbi_sock_dequeue_wr(csk);
+ credits -= p->csum;
+ kfree_skb(p);
+ }
+ }
+
+ cxgbi_sock_check_wr_invariants(csk);
+
+ if (seq_chk) {
+ if (unlikely(before(snd_una, csk->snd_una))) {
+ pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.",
+ csk, csk->state, csk->flags, csk->tid, snd_una,
+ csk->snd_una);
+ goto done;
+ }
+
+ if (csk->snd_una != snd_una) {
+ csk->snd_una = snd_una;
+ dst_confirm(csk->dst);
+ }
+ }
+
+ if (skb_queue_len(&csk->write_queue)) {
+ if (csk->cdev->csk_push_tx_frames(csk, 0))
+ cxgbi_conn_tx_open(csk);
+ } else
+ cxgbi_conn_tx_open(csk);
+done:
+ spin_unlock_bh(&csk->lock);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack);
+
+static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk,
+ unsigned short mtu)
+{
+ int i = 0;
+
+ while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu)
+ ++i;
+
+ return i;
+}
+
+unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu)
+{
+ unsigned int idx;
+ struct dst_entry *dst = csk->dst;
+
+ csk->advmss = dst_metric_advmss(dst);
+
+ if (csk->advmss > pmtu - 40)
+ csk->advmss = pmtu - 40;
+ if (csk->advmss < csk->cdev->mtus[0] - 40)
+ csk->advmss = csk->cdev->mtus[0] - 40;
+ idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40);
+
+ return idx;
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss);
+
+void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb)
+{
+ cxgbi_skcb_tcp_seq(skb) = csk->write_seq;
+ __skb_queue_tail(&csk->write_queue, skb);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail);
+
+void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb;
+
+ while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL)
+ kfree_skb(skb);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue);
+
+void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk)
+{
+ int pending = cxgbi_sock_count_pending_wrs(csk);
+
+ if (unlikely(csk->wr_cred + pending != csk->wr_max_cred))
+ pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
+ csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred);
+}
+EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants);
+
+static inline void
+scmd_get_params(struct scsi_cmnd *sc, struct scatterlist **sgl,
+ unsigned int *sgcnt, unsigned int *dlen,
+ unsigned int prot)
+{
+ struct scsi_data_buffer *sdb = prot ? scsi_prot(sc) : &sc->sdb;
+
+ *sgl = sdb->table.sgl;
+ *sgcnt = sdb->table.nents;
+ *dlen = sdb->length;
+ /* Caution: for protection sdb, sdb->length is invalid */
+}
+
+void cxgbi_ddp_set_one_ppod(struct cxgbi_pagepod *ppod,
+ struct cxgbi_task_tag_info *ttinfo,
+ struct scatterlist **sg_pp, unsigned int *sg_off)
+{
+ struct scatterlist *sg = sg_pp ? *sg_pp : NULL;
+ unsigned int offset = sg_off ? *sg_off : 0;
+ dma_addr_t addr = 0UL;
+ unsigned int len = 0;
+ int i;
+
+ memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr));
+
+ if (sg) {
+ addr = sg_dma_address(sg);
+ len = sg_dma_len(sg);
+ }
+
+ for (i = 0; i < PPOD_PAGES_MAX; i++) {
+ if (sg) {
+ ppod->addr[i] = cpu_to_be64(addr + offset);
+ offset += PAGE_SIZE;
+ if (offset == (len + sg->offset)) {
+ offset = 0;
+ sg = sg_next(sg);
+ if (sg) {
+ addr = sg_dma_address(sg);
+ len = sg_dma_len(sg);
+ }
+ }
+ } else {
+ ppod->addr[i] = 0ULL;
+ }
+ }
+
+ /*
+ * the fifth address needs to be repeated in the next ppod, so do
+ * not move sg
+ */
+ if (sg_pp) {
+ *sg_pp = sg;
+ *sg_off = offset;
+ }
+
+ if (offset == len) {
+ offset = 0;
+ sg = sg_next(sg);
+ if (sg) {
+ addr = sg_dma_address(sg);
+ len = sg_dma_len(sg);
+ }
+ }
+ ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL;
+}
+EXPORT_SYMBOL_GPL(cxgbi_ddp_set_one_ppod);
+
+/*
+ * APIs interacting with open-iscsi libraries
+ */
+
+int cxgbi_ddp_ppm_setup(void **ppm_pp, struct cxgbi_device *cdev,
+ struct cxgbi_tag_format *tformat,
+ unsigned int iscsi_size, unsigned int llimit,
+ unsigned int start, unsigned int rsvd_factor,
+ unsigned int edram_start, unsigned int edram_size)
+{
+ int err = cxgbi_ppm_init(ppm_pp, cdev->ports[0], cdev->pdev,
+ cdev->lldev, tformat, iscsi_size, llimit, start,
+ rsvd_factor, edram_start, edram_size);
+
+ if (err >= 0) {
+ struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*ppm_pp);
+
+ if (ppm->ppmax < 1024 ||
+ ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX)
+ cdev->flags |= CXGBI_FLAG_DDP_OFF;
+ err = 0;
+ } else {
+ cdev->flags |= CXGBI_FLAG_DDP_OFF;
+ }
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(cxgbi_ddp_ppm_setup);
+
+static int cxgbi_ddp_sgl_check(struct scatterlist *sgl, int nents)
+{
+ int i;
+ int last_sgidx = nents - 1;
+ struct scatterlist *sg = sgl;
+
+ for (i = 0; i < nents; i++, sg = sg_next(sg)) {
+ unsigned int len = sg->length + sg->offset;
+
+ if ((sg->offset & 0x3) || (i && sg->offset) ||
+ ((i != last_sgidx) && len != PAGE_SIZE)) {
+ log_debug(1 << CXGBI_DBG_DDP,
+ "sg %u/%u, %u,%u, not aligned.\n",
+ i, nents, sg->offset, sg->length);
+ goto err_out;
+ }
+ }
+ return 0;
+err_out:
+ return -EINVAL;
+}
+
+static int cxgbi_ddp_reserve(struct cxgbi_conn *cconn,
+ struct cxgbi_task_data *tdata, u32 sw_tag,
+ unsigned int xferlen)
+{
+ struct cxgbi_sock *csk = cconn->cep->csk;
+ struct cxgbi_device *cdev = csk->cdev;
+ struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
+ struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
+ struct scatterlist *sgl = ttinfo->sgl;
+ unsigned int sgcnt = ttinfo->nents;
+ unsigned int sg_offset = sgl->offset;
+ int err;
+
+ if (cdev->flags & CXGBI_FLAG_DDP_OFF) {
+ log_debug(1 << CXGBI_DBG_DDP,
+ "cdev 0x%p DDP off.\n", cdev);
+ return -EINVAL;
+ }
+
+ if (!ppm || xferlen < DDP_THRESHOLD || !sgcnt ||
+ ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX) {
+ log_debug(1 << CXGBI_DBG_DDP,
+ "ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n",
+ ppm, ppm ? ppm->tformat.pgsz_idx_dflt : DDP_PGIDX_MAX,
+ xferlen, ttinfo->nents);
+ return -EINVAL;
+ }
+
+ /* make sure the buffer is suitable for ddp */
+ if (cxgbi_ddp_sgl_check(sgl, sgcnt) < 0)
+ return -EINVAL;
+
+ ttinfo->nr_pages = (xferlen + sgl->offset + (1 << PAGE_SHIFT) - 1) >>
+ PAGE_SHIFT;
+
+ /*
+ * the ddp tag will be used for the itt in the outgoing pdu,
+ * the itt genrated by libiscsi is saved in the ppm and can be
+ * retrieved via the ddp tag
+ */
+ err = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx,
+ &ttinfo->tag, (unsigned long)sw_tag);
+ if (err < 0) {
+ cconn->ddp_full++;
+ return err;
+ }
+ ttinfo->npods = err;
+
+ /* setup dma from scsi command sgl */
+ sgl->offset = 0;
+ err = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
+ sgl->offset = sg_offset;
+ if (err == 0) {
+ pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n",
+ __func__, sw_tag, xferlen, sgcnt);
+ goto rel_ppods;
+ }
+ if (err != ttinfo->nr_pages) {
+ log_debug(1 << CXGBI_DBG_DDP,
+ "%s: sw tag 0x%x, xfer %u, sgl %u, dma count %d.\n",
+ __func__, sw_tag, xferlen, sgcnt, err);
+ }
+
+ ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_MAPPED;
+ ttinfo->cid = csk->port_id;
+
+ cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset,
+ xferlen, &ttinfo->hdr);
+
+ if (cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ) {
+ /* write ppod from xmit_pdu (of iscsi_scsi_command pdu) */
+ ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_VALID;
+ } else {
+ /* write ppod from control queue now */
+ err = cdev->csk_ddp_set_map(ppm, csk, ttinfo);
+ if (err < 0)
+ goto rel_ppods;
+ }
+
+ return 0;
+
+rel_ppods:
+ cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
+
+ if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_MAPPED) {
+ ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_MAPPED;
+ dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
+ }
+ return -EINVAL;
+}
+
+static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt)
+{
+ struct scsi_cmnd *sc = task->sc;
+ struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct cxgbi_device *cdev = cconn->chba->cdev;
+ struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
+ u32 tag = ntohl((__force u32)hdr_itt);
+
+ log_debug(1 << CXGBI_DBG_DDP,
+ "cdev 0x%p, task 0x%p, release tag 0x%x.\n",
+ cdev, task, tag);
+ if (sc && sc->sc_data_direction == DMA_FROM_DEVICE &&
+ cxgbi_ppm_is_ddp_tag(ppm, tag)) {
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
+
+ if (!(cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ))
+ cdev->csk_ddp_clear_map(cdev, ppm, ttinfo);
+ cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
+ dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, ttinfo->nents,
+ DMA_FROM_DEVICE);
+ }
+}
+
+static inline u32 cxgbi_build_sw_tag(u32 idx, u32 age)
+{
+ /* assume idx and age both are < 0x7FFF (32767) */
+ return (idx << 16) | age;
+}
+
+static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
+{
+ struct scsi_cmnd *sc = task->sc;
+ struct iscsi_conn *conn = task->conn;
+ struct iscsi_session *sess = conn->session;
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct cxgbi_device *cdev = cconn->chba->cdev;
+ struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
+ u32 sw_tag = cxgbi_build_sw_tag(task->itt, sess->age);
+ u32 tag = 0;
+ int err = -EINVAL;
+
+ if (sc && sc->sc_data_direction == DMA_FROM_DEVICE) {
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
+
+ scmd_get_params(sc, &ttinfo->sgl, &ttinfo->nents,
+ &tdata->dlen, 0);
+ err = cxgbi_ddp_reserve(cconn, tdata, sw_tag, tdata->dlen);
+ if (!err)
+ tag = ttinfo->tag;
+ else
+ log_debug(1 << CXGBI_DBG_DDP,
+ "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n",
+ cconn->cep->csk, task, tdata->dlen,
+ ttinfo->nents);
+ }
+
+ if (err < 0) {
+ err = cxgbi_ppm_make_non_ddp_tag(ppm, sw_tag, &tag);
+ if (err < 0)
+ return err;
+ }
+ /* the itt need to sent in big-endian order */
+ *hdr_itt = (__force itt_t)htonl(tag);
+
+ log_debug(1 << CXGBI_DBG_DDP,
+ "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n",
+ cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt);
+ return 0;
+}
+
+void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age)
+{
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct cxgbi_device *cdev = cconn->chba->cdev;
+ struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev);
+ u32 tag = ntohl((__force u32)itt);
+ u32 sw_bits;
+
+ if (ppm) {
+ if (cxgbi_ppm_is_ddp_tag(ppm, tag))
+ sw_bits = cxgbi_ppm_get_tag_caller_data(ppm, tag);
+ else
+ sw_bits = cxgbi_ppm_decode_non_ddp_tag(ppm, tag);
+ } else {
+ sw_bits = tag;
+ }
+
+ cxgbi_decode_sw_tag(sw_bits, idx, age);
+ log_debug(1 << CXGBI_DBG_DDP,
+ "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n",
+ cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
+ age ? *age : 0xFF);
+}
+EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt);
+
+void cxgbi_conn_tx_open(struct cxgbi_sock *csk)
+{
+ struct iscsi_conn *conn = csk->user_data;
+
+ if (conn) {
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "csk 0x%p, cid %d.\n", csk, conn->id);
+ iscsi_conn_queue_work(conn);
+ }
+}
+EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open);
+
+/*
+ * pdu receive, interact with libiscsi_tcp
+ */
+static inline int read_pdu_skb(struct iscsi_conn *conn,
+ struct sk_buff *skb,
+ unsigned int offset,
+ int offloaded)
+{
+ int status = 0;
+ int bytes_read;
+
+ bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
+ switch (status) {
+ case ISCSI_TCP_CONN_ERR:
+ pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n",
+ skb, offset, offloaded);
+ return -EIO;
+ case ISCSI_TCP_SUSPENDED:
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n",
+ skb, offset, offloaded, bytes_read);
+ /* no transfer - just have caller flush queue */
+ return bytes_read;
+ case ISCSI_TCP_SKB_DONE:
+ pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n",
+ skb, offset, offloaded);
+ /*
+ * pdus should always fit in the skb and we should get
+ * segment done notifcation.
+ */
+ iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
+ return -EFAULT;
+ case ISCSI_TCP_SEGMENT_DONE:
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n",
+ skb, offset, offloaded, bytes_read);
+ return bytes_read;
+ default:
+ pr_info("skb 0x%p, off %u, %d, invalid status %d.\n",
+ skb, offset, offloaded, status);
+ return -EINVAL;
+ }
+}
+
+static int
+skb_read_pdu_bhs(struct cxgbi_sock *csk, struct iscsi_conn *conn,
+ struct sk_buff *skb)
+{
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ int err;
+
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
+ conn, skb, skb->len, cxgbi_skcb_flags(skb));
+
+ if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
+ pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb);
+ iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
+ return -EIO;
+ }
+
+ if (conn->hdrdgst_en &&
+ cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) {
+ pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb);
+ iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
+ return -EIO;
+ }
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_RX_ISCSI_COMPL) &&
+ cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA_DDPD)) {
+ /* If completion flag is set and data is directly
+ * placed in to the host memory then update
+ * task->exp_datasn to the datasn in completion
+ * iSCSI hdr as T6 adapter generates completion only
+ * for the last pdu of a sequence.
+ */
+ itt_t itt = ((struct iscsi_data *)skb->data)->itt;
+ struct iscsi_task *task = iscsi_itt_to_ctask(conn, itt);
+ u32 data_sn = be32_to_cpu(((struct iscsi_data *)
+ skb->data)->datasn);
+ if (task && task->sc) {
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
+
+ tcp_task->exp_datasn = data_sn;
+ }
+ }
+
+ err = read_pdu_skb(conn, skb, 0, 0);
+ if (likely(err >= 0)) {
+ struct iscsi_hdr *hdr = (struct iscsi_hdr *)skb->data;
+ u8 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
+
+ if (unlikely(opcode == ISCSI_OP_LOGOUT_RSP))
+ cxgbi_sock_set_flag(csk, CTPF_LOGOUT_RSP_RCVD);
+ }
+
+ return err;
+}
+
+static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb,
+ struct sk_buff *skb, unsigned int offset)
+{
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ bool offloaded = 0;
+ int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
+
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
+ conn, skb, skb->len, cxgbi_skcb_flags(skb));
+
+ if (conn->datadgst_en &&
+ cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) {
+ pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n",
+ conn, lskb, cxgbi_skcb_flags(lskb));
+ iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
+ return -EIO;
+ }
+
+ if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
+ return 0;
+
+ /* coalesced, add header digest length */
+ if (lskb == skb && conn->hdrdgst_en)
+ offset += ISCSI_DIGEST_SIZE;
+
+ if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD))
+ offloaded = 1;
+
+ if (opcode == ISCSI_OP_SCSI_DATA_IN)
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n",
+ skb, opcode, ntohl(tcp_conn->in.hdr->itt),
+ tcp_conn->in.datalen, offloaded ? "is" : "not");
+
+ return read_pdu_skb(conn, skb, offset, offloaded);
+}
+
+static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ int must_send;
+ u32 credits;
+
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n",
+ csk, csk->state, csk->flags, csk->tid, csk->copied_seq,
+ csk->rcv_wup, cdev->rx_credit_thres,
+ csk->rcv_win);
+
+ if (!cdev->rx_credit_thres)
+ return;
+
+ if (csk->state != CTP_ESTABLISHED)
+ return;
+
+ credits = csk->copied_seq - csk->rcv_wup;
+ if (unlikely(!credits))
+ return;
+ must_send = credits + 16384 >= csk->rcv_win;
+ if (must_send || credits >= cdev->rx_credit_thres)
+ csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits);
+}
+
+void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct iscsi_conn *conn = csk->user_data;
+ struct sk_buff *skb;
+ unsigned int read = 0;
+ int err = 0;
+
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, conn 0x%p.\n", csk, conn);
+
+ if (unlikely(!conn || conn->suspend_rx)) {
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n",
+ csk, conn, conn ? conn->id : 0xFF,
+ conn ? conn->suspend_rx : 0xFF);
+ return;
+ }
+
+ while (!err) {
+ skb = skb_peek(&csk->receive_queue);
+ if (!skb ||
+ !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) {
+ if (skb)
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "skb 0x%p, NOT ready 0x%lx.\n",
+ skb, cxgbi_skcb_flags(skb));
+ break;
+ }
+ __skb_unlink(skb, &csk->receive_queue);
+
+ read += cxgbi_skcb_rx_pdulen(skb);
+ log_debug(1 << CXGBI_DBG_PDU_RX,
+ "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n",
+ csk, skb, skb->len, cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb));
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) {
+ err = skb_read_pdu_bhs(csk, conn, skb);
+ if (err < 0) {
+ pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, "
+ "f 0x%lx, plen %u.\n",
+ csk, skb, skb->len,
+ cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb));
+ goto skb_done;
+ }
+ err = skb_read_pdu_data(conn, skb, skb,
+ err + cdev->skb_rx_extra);
+ if (err < 0)
+ pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, "
+ "f 0x%lx, plen %u.\n",
+ csk, skb, skb->len,
+ cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb));
+ } else {
+ err = skb_read_pdu_bhs(csk, conn, skb);
+ if (err < 0) {
+ pr_err("bhs, csk 0x%p, skb 0x%p,%u, "
+ "f 0x%lx, plen %u.\n",
+ csk, skb, skb->len,
+ cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb));
+ goto skb_done;
+ }
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) {
+ struct sk_buff *dskb;
+
+ dskb = skb_peek(&csk->receive_queue);
+ if (!dskb) {
+ pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx,"
+ " plen %u, NO data.\n",
+ csk, skb, skb->len,
+ cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb));
+ err = -EIO;
+ goto skb_done;
+ }
+ __skb_unlink(dskb, &csk->receive_queue);
+
+ err = skb_read_pdu_data(conn, skb, dskb, 0);
+ if (err < 0)
+ pr_err("data, csk 0x%p, skb 0x%p,%u, "
+ "f 0x%lx, plen %u, dskb 0x%p,"
+ "%u.\n",
+ csk, skb, skb->len,
+ cxgbi_skcb_flags(skb),
+ cxgbi_skcb_rx_pdulen(skb),
+ dskb, dskb->len);
+ __kfree_skb(dskb);
+ } else
+ err = skb_read_pdu_data(conn, skb, skb, 0);
+ }
+skb_done:
+ __kfree_skb(skb);
+
+ if (err < 0)
+ break;
+ }
+
+ log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read);
+ if (read) {
+ csk->copied_seq += read;
+ csk_return_rx_credits(csk, read);
+ conn->rxdata_octets += read;
+ }
+
+ if (err < 0) {
+ pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n",
+ csk, conn, err, read);
+ iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
+ }
+}
+EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready);
+
+static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt,
+ unsigned int offset, unsigned int *off,
+ struct scatterlist **sgp)
+{
+ int i;
+ struct scatterlist *sg;
+
+ for_each_sg(sgl, sg, sgcnt, i) {
+ if (offset < sg->length) {
+ *off = offset;
+ *sgp = sg;
+ return 0;
+ }
+ offset -= sg->length;
+ }
+ return -EFAULT;
+}
+
+static int
+sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
+ unsigned int dlen, struct page_frag *frags,
+ int frag_max, u32 *dlimit)
+{
+ unsigned int datalen = dlen;
+ unsigned int sglen = sg->length - sgoffset;
+ struct page *page = sg_page(sg);
+ int i;
+
+ i = 0;
+ do {
+ unsigned int copy;
+
+ if (!sglen) {
+ sg = sg_next(sg);
+ if (!sg) {
+ pr_warn("sg %d NULL, len %u/%u.\n",
+ i, datalen, dlen);
+ return -EINVAL;
+ }
+ sgoffset = 0;
+ sglen = sg->length;
+ page = sg_page(sg);
+
+ }
+ copy = min(datalen, sglen);
+ if (i && page == frags[i - 1].page &&
+ sgoffset + sg->offset ==
+ frags[i - 1].offset + frags[i - 1].size) {
+ frags[i - 1].size += copy;
+ } else {
+ if (i >= frag_max) {
+ pr_warn("too many pages %u, dlen %u.\n",
+ frag_max, dlen);
+ *dlimit = dlen - datalen;
+ return -EINVAL;
+ }
+
+ frags[i].page = page;
+ frags[i].offset = sg->offset + sgoffset;
+ frags[i].size = copy;
+ i++;
+ }
+ datalen -= copy;
+ sgoffset += copy;
+ sglen -= copy;
+ } while (datalen);
+
+ return i;
+}
+
+static void cxgbi_task_data_sgl_check(struct iscsi_task *task)
+{
+ struct scsi_cmnd *sc = task->sc;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct scatterlist *sg, *sgl = NULL;
+ u32 sgcnt = 0;
+ int i;
+
+ tdata->flags = CXGBI_TASK_SGL_CHECKED;
+ if (!sc)
+ return;
+
+ scmd_get_params(sc, &sgl, &sgcnt, &tdata->dlen, 0);
+ if (!sgl || !sgcnt) {
+ tdata->flags |= CXGBI_TASK_SGL_COPY;
+ return;
+ }
+
+ for_each_sg(sgl, sg, sgcnt, i) {
+ if (page_count(sg_page(sg)) < 1) {
+ tdata->flags |= CXGBI_TASK_SGL_COPY;
+ return;
+ }
+ }
+}
+
+static int
+cxgbi_task_data_sgl_read(struct iscsi_task *task, u32 offset, u32 count,
+ u32 *dlimit)
+{
+ struct scsi_cmnd *sc = task->sc;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct scatterlist *sgl = NULL;
+ struct scatterlist *sg;
+ u32 dlen = 0;
+ u32 sgcnt;
+ int err;
+
+ if (!sc)
+ return 0;
+
+ scmd_get_params(sc, &sgl, &sgcnt, &dlen, 0);
+ if (!sgl || !sgcnt)
+ return 0;
+
+ err = sgl_seek_offset(sgl, sgcnt, offset, &tdata->sgoffset, &sg);
+ if (err < 0) {
+ pr_warn("tpdu max, sgl %u, bad offset %u/%u.\n",
+ sgcnt, offset, tdata->dlen);
+ return err;
+ }
+ err = sgl_read_to_frags(sg, tdata->sgoffset, count,
+ tdata->frags, MAX_SKB_FRAGS, dlimit);
+ if (err < 0) {
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "sgl max limit, sgl %u, offset %u, %u/%u, dlimit %u.\n",
+ sgcnt, offset, count, tdata->dlen, *dlimit);
+ return err;
+ }
+ tdata->offset = offset;
+ tdata->count = count;
+ tdata->nr_frags = err;
+ tdata->total_count = count;
+ tdata->total_offset = offset;
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "%s: offset %u, count %u,\n"
+ "err %u, total_count %u, total_offset %u\n",
+ __func__, offset, count, err, tdata->total_count, tdata->total_offset);
+
+ return 0;
+}
+
+int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 op)
+{
+ struct iscsi_conn *conn = task->conn;
+ struct iscsi_session *session = task->conn->session;
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct cxgbi_device *cdev = cconn->chba->cdev;
+ struct cxgbi_sock *csk = cconn->cep ? cconn->cep->csk : NULL;
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct scsi_cmnd *sc = task->sc;
+ u32 headroom = SKB_TX_ISCSI_PDU_HEADER_MAX;
+ u32 max_txdata_len = conn->max_xmit_dlength;
+ u32 iso_tx_rsvd = 0, local_iso_info = 0;
+ u32 last_tdata_offset, last_tdata_count;
+ int err = 0;
+
+ if (!tcp_task) {
+ pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p.\n",
+ task, tcp_task, tdata);
+ return -ENOMEM;
+ }
+ if (!csk) {
+ pr_err("task 0x%p, csk gone.\n", task);
+ return -EPIPE;
+ }
+
+ op &= ISCSI_OPCODE_MASK;
+
+ tcp_task->dd_data = tdata;
+ task->hdr = NULL;
+
+ last_tdata_count = tdata->count;
+ last_tdata_offset = tdata->offset;
+
+ if ((op == ISCSI_OP_SCSI_DATA_OUT) ||
+ ((op == ISCSI_OP_SCSI_CMD) &&
+ (sc->sc_data_direction == DMA_TO_DEVICE))) {
+ u32 remaining_data_tosend, dlimit = 0;
+ u32 max_pdu_size, max_num_pdu, num_pdu;
+ u32 count;
+
+ /* Preserve conn->max_xmit_dlength because it can get updated to
+ * ISO data size.
+ */
+ if (task->state == ISCSI_TASK_PENDING)
+ tdata->max_xmit_dlength = conn->max_xmit_dlength;
+
+ if (!tdata->offset)
+ cxgbi_task_data_sgl_check(task);
+
+ remaining_data_tosend =
+ tdata->dlen - tdata->offset - tdata->count;
+
+recalculate_sgl:
+ max_txdata_len = tdata->max_xmit_dlength;
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "tdata->dlen %u, remaining to send %u "
+ "conn->max_xmit_dlength %u, "
+ "tdata->max_xmit_dlength %u\n",
+ tdata->dlen, remaining_data_tosend,
+ conn->max_xmit_dlength, tdata->max_xmit_dlength);
+
+ if (cdev->skb_iso_txhdr && !csk->disable_iso &&
+ (remaining_data_tosend > tdata->max_xmit_dlength) &&
+ !(remaining_data_tosend % 4)) {
+ u32 max_iso_data;
+
+ if ((op == ISCSI_OP_SCSI_CMD) &&
+ session->initial_r2t_en)
+ goto no_iso;
+
+ max_pdu_size = tdata->max_xmit_dlength +
+ ISCSI_PDU_NONPAYLOAD_LEN;
+ max_iso_data = rounddown(CXGBI_MAX_ISO_DATA_IN_SKB,
+ csk->advmss);
+ max_num_pdu = max_iso_data / max_pdu_size;
+
+ num_pdu = (remaining_data_tosend +
+ tdata->max_xmit_dlength - 1) /
+ tdata->max_xmit_dlength;
+
+ if (num_pdu > max_num_pdu)
+ num_pdu = max_num_pdu;
+
+ conn->max_xmit_dlength = tdata->max_xmit_dlength * num_pdu;
+ max_txdata_len = conn->max_xmit_dlength;
+ iso_tx_rsvd = cdev->skb_iso_txhdr;
+ local_iso_info = sizeof(struct cxgbi_iso_info);
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "max_pdu_size %u, max_num_pdu %u, "
+ "max_txdata %u, num_pdu %u\n",
+ max_pdu_size, max_num_pdu,
+ max_txdata_len, num_pdu);
+ }
+no_iso:
+ count = min_t(u32, max_txdata_len, remaining_data_tosend);
+ err = cxgbi_task_data_sgl_read(task,
+ tdata->offset + tdata->count,
+ count, &dlimit);
+ if (unlikely(err < 0)) {
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "task 0x%p, tcp_task 0x%p, tdata 0x%p, "
+ "sgl err %d, count %u, dlimit %u\n",
+ task, tcp_task, tdata, err, count, dlimit);
+ if (dlimit) {
+ remaining_data_tosend =
+ rounddown(dlimit,
+ tdata->max_xmit_dlength);
+ if (!remaining_data_tosend)
+ remaining_data_tosend = dlimit;
+
+ dlimit = 0;
+
+ conn->max_xmit_dlength = remaining_data_tosend;
+ goto recalculate_sgl;
+ }
+
+ pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p, "
+ "sgl err %d\n",
+ task, tcp_task, tdata, err);
+ goto ret_err;
+ }
+
+ if ((tdata->flags & CXGBI_TASK_SGL_COPY) ||
+ (tdata->nr_frags > MAX_SKB_FRAGS))
+ headroom += conn->max_xmit_dlength;
+ }
+
+ tdata->skb = alloc_skb(local_iso_info + cdev->skb_tx_rsvd +
+ iso_tx_rsvd + headroom, GFP_ATOMIC);
+ if (!tdata->skb) {
+ tdata->count = last_tdata_count;
+ tdata->offset = last_tdata_offset;
+ err = -ENOMEM;
+ goto ret_err;
+ }
+
+ skb_reserve(tdata->skb, local_iso_info + cdev->skb_tx_rsvd +
+ iso_tx_rsvd);
+
+ if (task->sc) {
+ task->hdr = (struct iscsi_hdr *)tdata->skb->data;
+ } else {
+ task->hdr = kzalloc(SKB_TX_ISCSI_PDU_HEADER_MAX, GFP_ATOMIC);
+ if (!task->hdr) {
+ __kfree_skb(tdata->skb);
+ tdata->skb = NULL;
+ return -ENOMEM;
+ }
+ }
+
+ task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX;
+
+ if (iso_tx_rsvd)
+ cxgbi_skcb_set_flag(tdata->skb, SKCBF_TX_ISO);
+
+ /* data_out uses scsi_cmd's itt */
+ if (op != ISCSI_OP_SCSI_DATA_OUT)
+ task_reserve_itt(task, &task->hdr->itt);
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n",
+ task, op, tdata->skb, cdev->skb_tx_rsvd, headroom,
+ conn->max_xmit_dlength, be32_to_cpu(task->hdr->itt));
+
+ return 0;
+
+ret_err:
+ conn->max_xmit_dlength = tdata->max_xmit_dlength;
+ return err;
+}
+EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu);
+
+static int
+cxgbi_prep_iso_info(struct iscsi_task *task, struct sk_buff *skb,
+ u32 count)
+{
+ struct cxgbi_iso_info *iso_info = (struct cxgbi_iso_info *)skb->head;
+ struct iscsi_r2t_info *r2t;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct iscsi_conn *conn = task->conn;
+ struct iscsi_session *session = conn->session;
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
+ u32 burst_size = 0, r2t_dlength = 0, dlength;
+ u32 max_pdu_len = tdata->max_xmit_dlength;
+ u32 segment_offset = 0;
+ u32 num_pdu;
+
+ if (unlikely(!cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)))
+ return 0;
+
+ memset(iso_info, 0, sizeof(struct cxgbi_iso_info));
+
+ if (task->hdr->opcode == ISCSI_OP_SCSI_CMD && session->imm_data_en) {
+ iso_info->flags |= CXGBI_ISO_INFO_IMM_ENABLE;
+ burst_size = count;
+ }
+
+ dlength = ntoh24(task->hdr->dlength);
+ dlength = min(dlength, max_pdu_len);
+ hton24(task->hdr->dlength, dlength);
+
+ num_pdu = (count + max_pdu_len - 1) / max_pdu_len;
+
+ if (iscsi_task_has_unsol_data(task))
+ r2t = &task->unsol_r2t;
+ else
+ r2t = tcp_task->r2t;
+
+ if (r2t) {
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "count %u, tdata->count %u, num_pdu %u,"
+ "task->hdr_len %u, r2t->data_length %u, r2t->sent %u\n",
+ count, tdata->count, num_pdu, task->hdr_len,
+ r2t->data_length, r2t->sent);
+
+ r2t_dlength = r2t->data_length - r2t->sent;
+ segment_offset = r2t->sent;
+ r2t->datasn += num_pdu - 1;
+ }
+
+ if (!r2t || !r2t->sent)
+ iso_info->flags |= CXGBI_ISO_INFO_FSLICE;
+
+ if (task->hdr->flags & ISCSI_FLAG_CMD_FINAL)
+ iso_info->flags |= CXGBI_ISO_INFO_LSLICE;
+
+ task->hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
+
+ iso_info->op = task->hdr->opcode;
+ iso_info->ahs = task->hdr->hlength;
+ iso_info->num_pdu = num_pdu;
+ iso_info->mpdu = max_pdu_len;
+ iso_info->burst_size = (burst_size + r2t_dlength) >> 2;
+ iso_info->len = count + task->hdr_len;
+ iso_info->segment_offset = segment_offset;
+
+ cxgbi_skcb_tx_iscsi_hdrlen(skb) = task->hdr_len;
+ return 0;
+}
+
+static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
+{
+ if (hcrc || dcrc) {
+ u8 submode = 0;
+
+ if (hcrc)
+ submode |= 1;
+ if (dcrc)
+ submode |= 2;
+ cxgbi_skcb_tx_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode;
+ } else
+ cxgbi_skcb_tx_ulp_mode(skb) = 0;
+}
+
+static struct page *rsvd_page;
+
+int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
+ unsigned int count)
+{
+ struct iscsi_conn *conn = task->conn;
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct sk_buff *skb;
+ struct scsi_cmnd *sc = task->sc;
+ u32 expected_count, expected_offset;
+ u32 datalen = count, dlimit = 0;
+ u32 i, padlen = iscsi_padding(count);
+ struct page *pg;
+ int err;
+
+ if (!tcp_task || (tcp_task->dd_data != tdata)) {
+ pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
+ task, task->sc, tcp_task,
+ tcp_task ? tcp_task->dd_data : NULL, tdata);
+ return -EINVAL;
+ }
+ skb = tdata->skb;
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n",
+ task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK,
+ be32_to_cpu(task->cmdsn), be32_to_cpu(task->hdr->itt), offset, count);
+
+ skb_put(skb, task->hdr_len);
+ tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
+ if (!count) {
+ tdata->count = count;
+ tdata->offset = offset;
+ tdata->nr_frags = 0;
+ tdata->total_offset = 0;
+ tdata->total_count = 0;
+ if (tdata->max_xmit_dlength)
+ conn->max_xmit_dlength = tdata->max_xmit_dlength;
+ cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO);
+ return 0;
+ }
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "data->total_count %u, tdata->total_offset %u\n",
+ tdata->total_count, tdata->total_offset);
+
+ expected_count = tdata->total_count;
+ expected_offset = tdata->total_offset;
+
+ if ((count != expected_count) ||
+ (offset != expected_offset)) {
+ err = cxgbi_task_data_sgl_read(task, offset, count, &dlimit);
+ if (err < 0) {
+ pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p "
+ "dlimit %u, sgl err %d.\n", task, task->sc,
+ tcp_task, tcp_task ? tcp_task->dd_data : NULL,
+ tdata, dlimit, err);
+ return err;
+ }
+ }
+
+ /* Restore original value of conn->max_xmit_dlength because
+ * it can get updated to ISO data size.
+ */
+ conn->max_xmit_dlength = tdata->max_xmit_dlength;
+
+ if (sc) {
+ struct page_frag *frag = tdata->frags;
+
+ if ((tdata->flags & CXGBI_TASK_SGL_COPY) ||
+ (tdata->nr_frags > MAX_SKB_FRAGS) ||
+ (padlen && (tdata->nr_frags ==
+ MAX_SKB_FRAGS))) {
+ char *dst = skb->data + task->hdr_len;
+
+ /* data fits in the skb's headroom */
+ for (i = 0; i < tdata->nr_frags; i++, frag++) {
+ char *src = kmap_atomic(frag->page);
+
+ memcpy(dst, src + frag->offset, frag->size);
+ dst += frag->size;
+ kunmap_atomic(src);
+ }
+
+ if (padlen) {
+ memset(dst, 0, padlen);
+ padlen = 0;
+ }
+ skb_put(skb, count + padlen);
+ } else {
+ for (i = 0; i < tdata->nr_frags; i++, frag++) {
+ get_page(frag->page);
+ skb_fill_page_desc(skb, i, frag->page,
+ frag->offset, frag->size);
+ }
+
+ skb->len += count;
+ skb->data_len += count;
+ skb->truesize += count;
+ }
+ } else {
+ pg = virt_to_head_page(task->data);
+ get_page(pg);
+ skb_fill_page_desc(skb, 0, pg,
+ task->data - (char *)page_address(pg),
+ count);
+ skb->len += count;
+ skb->data_len += count;
+ skb->truesize += count;
+ }
+
+ if (padlen) {
+ get_page(rsvd_page);
+ skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
+ rsvd_page, 0, padlen);
+
+ skb->data_len += padlen;
+ skb->truesize += padlen;
+ skb->len += padlen;
+ }
+
+ if (likely(count > tdata->max_xmit_dlength))
+ cxgbi_prep_iso_info(task, skb, count);
+ else
+ cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu);
+
+static int cxgbi_sock_tx_queue_up(struct cxgbi_sock *csk, struct sk_buff *skb)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ struct cxgbi_iso_info *iso_cpl;
+ u32 frags = skb_shinfo(skb)->nr_frags;
+ u32 extra_len, num_pdu, hdr_len;
+ u32 iso_tx_rsvd = 0;
+
+ if (csk->state != CTP_ESTABLISHED) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n",
+ csk, csk->state, csk->flags, csk->tid);
+ return -EPIPE;
+ }
+
+ if (csk->err) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n",
+ csk, csk->state, csk->flags, csk->tid, csk->err);
+ return -EPIPE;
+ }
+
+ if ((cdev->flags & CXGBI_FLAG_DEV_T3) &&
+ before((csk->snd_win + csk->snd_una), csk->write_seq)) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n",
+ csk, csk->state, csk->flags, csk->tid, csk->write_seq,
+ csk->snd_una, csk->snd_win);
+ return -ENOBUFS;
+ }
+
+ if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))
+ iso_tx_rsvd = cdev->skb_iso_txhdr;
+
+ if (unlikely(skb_headroom(skb) < (cdev->skb_tx_rsvd + iso_tx_rsvd))) {
+ pr_err("csk 0x%p, skb head %u < %u.\n",
+ csk, skb_headroom(skb), cdev->skb_tx_rsvd);
+ return -EINVAL;
+ }
+
+ if (skb->len != skb->data_len)
+ frags++;
+
+ if (frags >= SKB_WR_LIST_SIZE) {
+ pr_err("csk 0x%p, frags %u, %u,%u >%lu.\n",
+ csk, skb_shinfo(skb)->nr_frags, skb->len,
+ skb->data_len, SKB_WR_LIST_SIZE);
+ return -EINVAL;
+ }
+
+ cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR);
+ skb_reset_transport_header(skb);
+ cxgbi_sock_skb_entail(csk, skb);
+
+ extra_len = cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));
+
+ if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))) {
+ iso_cpl = (struct cxgbi_iso_info *)skb->head;
+ num_pdu = iso_cpl->num_pdu;
+ hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb);
+ extra_len = (cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb)) *
+ num_pdu) + (hdr_len * (num_pdu - 1));
+ }
+
+ csk->write_seq += (skb->len + extra_len);
+
+ return 0;
+}
+
+static int cxgbi_sock_send_skb(struct cxgbi_sock *csk, struct sk_buff *skb)
+{
+ struct cxgbi_device *cdev = csk->cdev;
+ int len = skb->len;
+ int err;
+
+ spin_lock_bh(&csk->lock);
+ err = cxgbi_sock_tx_queue_up(csk, skb);
+ if (err < 0) {
+ spin_unlock_bh(&csk->lock);
+ return err;
+ }
+
+ if (likely(skb_queue_len(&csk->write_queue)))
+ cdev->csk_push_tx_frames(csk, 0);
+ spin_unlock_bh(&csk->lock);
+ return len;
+}
+
+int cxgbi_conn_xmit_pdu(struct iscsi_task *task)
+{
+ struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+ struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo;
+ struct sk_buff *skb;
+ struct cxgbi_sock *csk = NULL;
+ u32 pdulen = 0;
+ u32 datalen;
+ int err;
+
+ if (!tcp_task || (tcp_task->dd_data != tdata)) {
+ pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
+ task, task->sc, tcp_task,
+ tcp_task ? tcp_task->dd_data : NULL, tdata);
+ return -EINVAL;
+ }
+
+ skb = tdata->skb;
+ if (!skb) {
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "task 0x%p, skb NULL.\n", task);
+ return 0;
+ }
+
+ if (cconn && cconn->cep)
+ csk = cconn->cep->csk;
+
+ if (!csk) {
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "task 0x%p, csk gone.\n", task);
+ return -EPIPE;
+ }
+
+ tdata->skb = NULL;
+ datalen = skb->data_len;
+
+ /* write ppod first if using ofldq to write ppod */
+ if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_VALID) {
+ struct cxgbi_ppm *ppm = csk->cdev->cdev2ppm(csk->cdev);
+
+ ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_VALID;
+ if (csk->cdev->csk_ddp_set_map(ppm, csk, ttinfo) < 0)
+ pr_err("task 0x%p, ppod writing using ofldq failed.\n",
+ task);
+ /* continue. Let fl get the data */
+ }
+
+ if (!task->sc)
+ memcpy(skb->data, task->hdr, SKB_TX_ISCSI_PDU_HEADER_MAX);
+
+ err = cxgbi_sock_send_skb(csk, skb);
+ if (err > 0) {
+ pdulen += err;
+
+ log_debug(1 << CXGBI_DBG_PDU_TX, "task 0x%p,0x%p, rv %d.\n",
+ task, task->sc, err);
+
+ if (task->conn->hdrdgst_en)
+ pdulen += ISCSI_DIGEST_SIZE;
+
+ if (datalen && task->conn->datadgst_en)
+ pdulen += ISCSI_DIGEST_SIZE;
+
+ task->conn->txdata_octets += pdulen;
+
+ if (unlikely(cxgbi_is_iso_config(csk) && cxgbi_is_iso_disabled(csk))) {
+ if (time_after(jiffies, csk->prev_iso_ts + HZ)) {
+ csk->disable_iso = false;
+ csk->prev_iso_ts = 0;
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "enable iso: csk 0x%p\n", csk);
+ }
+ }
+
+ return 0;
+ }
+
+ if (err == -EAGAIN || err == -ENOBUFS) {
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n",
+ task, skb, skb->len, skb->data_len, err);
+ /* reset skb to send when we are called again */
+ tdata->skb = skb;
+
+ if (cxgbi_is_iso_config(csk) && !cxgbi_is_iso_disabled(csk) &&
+ (csk->no_tx_credits++ >= 2)) {
+ csk->disable_iso = true;
+ csk->prev_iso_ts = jiffies;
+ log_debug(1 << CXGBI_DBG_PDU_TX,
+ "disable iso:csk 0x%p, ts:%lu\n",
+ csk, csk->prev_iso_ts);
+ }
+
+ return err;
+ }
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
+ "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
+ task->itt, skb, skb->len, skb->data_len, err);
+ __kfree_skb(skb);
+ iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
+ iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
+ return err;
+}
+EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu);
+
+void cxgbi_cleanup_task(struct iscsi_task *task)
+{
+ struct iscsi_tcp_task *tcp_task = task->dd_data;
+ struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
+
+ if (!tcp_task || (tcp_task->dd_data != tdata)) {
+ pr_info("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n",
+ task, task->sc, tcp_task,
+ tcp_task ? tcp_task->dd_data : NULL, tdata);
+ return;
+ }
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "task 0x%p, skb 0x%p, itt 0x%x.\n",
+ task, tdata->skb, task->hdr_itt);
+
+ tcp_task->dd_data = NULL;
+
+ if (!task->sc)
+ kfree(task->hdr);
+ task->hdr = NULL;
+
+ /* never reached the xmit task callout */
+ if (tdata->skb) {
+ __kfree_skb(tdata->skb);
+ tdata->skb = NULL;
+ }
+
+ task_release_itt(task, task->hdr_itt);
+ memset(tdata, 0, sizeof(*tdata));
+
+ iscsi_tcp_cleanup_task(task);
+}
+EXPORT_SYMBOL_GPL(cxgbi_cleanup_task);
+
+void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn,
+ struct iscsi_stats *stats)
+{
+ struct iscsi_conn *conn = cls_conn->dd_data;
+
+ stats->txdata_octets = conn->txdata_octets;
+ stats->rxdata_octets = conn->rxdata_octets;
+ stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
+ stats->dataout_pdus = conn->dataout_pdus_cnt;
+ stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
+ stats->datain_pdus = conn->datain_pdus_cnt;
+ stats->r2t_pdus = conn->r2t_pdus_cnt;
+ stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
+ stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
+ stats->digest_err = 0;
+ stats->timeout_err = 0;
+ stats->custom_length = 1;
+ strcpy(stats->custom[0].desc, "eh_abort_cnt");
+ stats->custom[0].value = conn->eh_abort_cnt;
+}
+EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats);
+
+static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn)
+{
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct cxgbi_device *cdev = cconn->chba->cdev;
+ unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd);
+ unsigned int max_def = 512 * MAX_SKB_FRAGS;
+ unsigned int max = max(max_def, headroom);
+
+ max = min(cconn->chba->cdev->tx_max_size, max);
+ if (conn->max_xmit_dlength)
+ conn->max_xmit_dlength = min(conn->max_xmit_dlength, max);
+ else
+ conn->max_xmit_dlength = max;
+ cxgbi_align_pdu_size(conn->max_xmit_dlength);
+
+ return 0;
+}
+
+static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn)
+{
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ unsigned int max = cconn->chba->cdev->rx_max_size;
+
+ cxgbi_align_pdu_size(max);
+
+ if (conn->max_recv_dlength) {
+ if (conn->max_recv_dlength > max) {
+ pr_err("MaxRecvDataSegmentLength %u > %u.\n",
+ conn->max_recv_dlength, max);
+ return -EINVAL;
+ }
+ conn->max_recv_dlength = min(conn->max_recv_dlength, max);
+ cxgbi_align_pdu_size(conn->max_recv_dlength);
+ } else
+ conn->max_recv_dlength = max;
+
+ return 0;
+}
+
+int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn,
+ enum iscsi_param param, char *buf, int buflen)
+{
+ struct iscsi_conn *conn = cls_conn->dd_data;
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct cxgbi_sock *csk = cconn->cep->csk;
+ int err;
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "cls_conn 0x%p, param %d, buf(%d) %s.\n",
+ cls_conn, param, buflen, buf);
+
+ switch (param) {
+ case ISCSI_PARAM_HDRDGST_EN:
+ err = iscsi_set_param(cls_conn, param, buf, buflen);
+ if (!err && conn->hdrdgst_en)
+ err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
+ conn->hdrdgst_en,
+ conn->datadgst_en);
+ break;
+ case ISCSI_PARAM_DATADGST_EN:
+ err = iscsi_set_param(cls_conn, param, buf, buflen);
+ if (!err && conn->datadgst_en)
+ err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
+ conn->hdrdgst_en,
+ conn->datadgst_en);
+ break;
+ case ISCSI_PARAM_MAX_R2T:
+ return iscsi_tcp_set_max_r2t(conn, buf);
+ case ISCSI_PARAM_MAX_RECV_DLENGTH:
+ err = iscsi_set_param(cls_conn, param, buf, buflen);
+ if (!err)
+ err = cxgbi_conn_max_recv_dlength(conn);
+ break;
+ case ISCSI_PARAM_MAX_XMIT_DLENGTH:
+ err = iscsi_set_param(cls_conn, param, buf, buflen);
+ if (!err)
+ err = cxgbi_conn_max_xmit_dlength(conn);
+ break;
+ default:
+ return iscsi_set_param(cls_conn, param, buf, buflen);
+ }
+ return err;
+}
+EXPORT_SYMBOL_GPL(cxgbi_set_conn_param);
+
+int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param,
+ char *buf)
+{
+ struct cxgbi_endpoint *cep = ep->dd_data;
+ struct cxgbi_sock *csk;
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "cls_conn 0x%p, param %d.\n", ep, param);
+
+ switch (param) {
+ case ISCSI_PARAM_CONN_PORT:
+ case ISCSI_PARAM_CONN_ADDRESS:
+ if (!cep)
+ return -ENOTCONN;
+
+ csk = cep->csk;
+ if (!csk)
+ return -ENOTCONN;
+
+ return iscsi_conn_get_addr_param((struct sockaddr_storage *)
+ &csk->daddr, param, buf);
+ default:
+ break;
+ }
+ return -ENOSYS;
+}
+EXPORT_SYMBOL_GPL(cxgbi_get_ep_param);
+
+struct iscsi_cls_conn *
+cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid)
+{
+ struct iscsi_cls_conn *cls_conn;
+ struct iscsi_conn *conn;
+ struct iscsi_tcp_conn *tcp_conn;
+ struct cxgbi_conn *cconn;
+
+ cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid);
+ if (!cls_conn)
+ return NULL;
+
+ conn = cls_conn->dd_data;
+ tcp_conn = conn->dd_data;
+ cconn = tcp_conn->dd_data;
+ cconn->iconn = conn;
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n",
+ cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn);
+
+ return cls_conn;
+}
+EXPORT_SYMBOL_GPL(cxgbi_create_conn);
+
+int cxgbi_bind_conn(struct iscsi_cls_session *cls_session,
+ struct iscsi_cls_conn *cls_conn,
+ u64 transport_eph, int is_leading)
+{
+ struct iscsi_conn *conn = cls_conn->dd_data;
+ struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+ struct cxgbi_conn *cconn = tcp_conn->dd_data;
+ struct cxgbi_ppm *ppm;
+ struct iscsi_endpoint *ep;
+ struct cxgbi_endpoint *cep;
+ struct cxgbi_sock *csk;
+ int err;
+
+ ep = iscsi_lookup_endpoint(transport_eph);
+ if (!ep)
+ return -EINVAL;
+
+ /* setup ddp pagesize */
+ cep = ep->dd_data;
+ csk = cep->csk;
+
+ ppm = csk->cdev->cdev2ppm(csk->cdev);
+ err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid,
+ ppm->tformat.pgsz_idx_dflt);
+ if (err < 0)
+ goto put_ep;
+
+ err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
+ if (err) {
+ err = -EINVAL;
+ goto put_ep;
+ }
+
+ /* calculate the tag idx bits needed for this conn based on cmds_max */
+ cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1;
+
+ write_lock_bh(&csk->callback_lock);
+ csk->user_data = conn;
+ cconn->chba = cep->chba;
+ cconn->cep = cep;
+ cep->cconn = cconn;
+ write_unlock_bh(&csk->callback_lock);
+
+ cxgbi_conn_max_xmit_dlength(conn);
+ cxgbi_conn_max_recv_dlength(conn);
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n",
+ cls_session, cls_conn, ep, cconn, csk);
+ /* init recv engine */
+ iscsi_tcp_hdr_recv_prep(tcp_conn);
+
+put_ep:
+ iscsi_put_endpoint(ep);
+ return err;
+}
+EXPORT_SYMBOL_GPL(cxgbi_bind_conn);
+
+struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep,
+ u16 cmds_max, u16 qdepth,
+ u32 initial_cmdsn)
+{
+ struct cxgbi_endpoint *cep;
+ struct cxgbi_hba *chba;
+ struct Scsi_Host *shost;
+ struct iscsi_cls_session *cls_session;
+ struct iscsi_session *session;
+
+ if (!ep) {
+ pr_err("missing endpoint.\n");
+ return NULL;
+ }
+
+ cep = ep->dd_data;
+ chba = cep->chba;
+ shost = chba->shost;
+
+ BUG_ON(chba != iscsi_host_priv(shost));
+
+ cls_session = iscsi_session_setup(chba->cdev->itp, shost,
+ cmds_max, 0,
+ sizeof(struct iscsi_tcp_task) +
+ sizeof(struct cxgbi_task_data),
+ initial_cmdsn, ISCSI_MAX_TARGET);
+ if (!cls_session)
+ return NULL;
+
+ session = cls_session->dd_data;
+ if (iscsi_tcp_r2tpool_alloc(session))
+ goto remove_session;
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "ep 0x%p, cls sess 0x%p.\n", ep, cls_session);
+ return cls_session;
+
+remove_session:
+ iscsi_session_teardown(cls_session);
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(cxgbi_create_session);
+
+void cxgbi_destroy_session(struct iscsi_cls_session *cls_session)
+{
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "cls sess 0x%p.\n", cls_session);
+
+ iscsi_tcp_r2tpool_free(cls_session->dd_data);
+ iscsi_session_teardown(cls_session);
+}
+EXPORT_SYMBOL_GPL(cxgbi_destroy_session);
+
+int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
+ char *buf, int buflen)
+{
+ struct cxgbi_hba *chba = iscsi_host_priv(shost);
+
+ if (!chba->ndev) {
+ shost_printk(KERN_ERR, shost, "Could not get host param. "
+ "netdev for host not set.\n");
+ return -ENODEV;
+ }
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n",
+ shost, chba, chba->ndev->name, param, buflen, buf);
+
+ switch (param) {
+ case ISCSI_HOST_PARAM_IPADDRESS:
+ {
+ __be32 addr = in_aton(buf);
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr);
+ cxgbi_set_iscsi_ipv4(chba, addr);
+ return 0;
+ }
+ case ISCSI_HOST_PARAM_HWADDRESS:
+ case ISCSI_HOST_PARAM_NETDEV_NAME:
+ return 0;
+ default:
+ return iscsi_host_set_param(shost, param, buf, buflen);
+ }
+}
+EXPORT_SYMBOL_GPL(cxgbi_set_host_param);
+
+int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
+ char *buf)
+{
+ struct cxgbi_hba *chba = iscsi_host_priv(shost);
+ int len = 0;
+
+ if (!chba->ndev) {
+ shost_printk(KERN_ERR, shost, "Could not get host param. "
+ "netdev for host not set.\n");
+ return -ENODEV;
+ }
+
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "shost 0x%p, hba 0x%p,%s, param %d.\n",
+ shost, chba, chba->ndev->name, param);
+
+ switch (param) {
+ case ISCSI_HOST_PARAM_HWADDRESS:
+ len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6);
+ break;
+ case ISCSI_HOST_PARAM_NETDEV_NAME:
+ len = sprintf(buf, "%s\n", chba->ndev->name);
+ break;
+ case ISCSI_HOST_PARAM_IPADDRESS:
+ {
+ struct cxgbi_sock *csk = find_sock_on_port(chba->cdev,
+ chba->port_id);
+ if (csk) {
+ len = sprintf(buf, "%pIS",
+ (struct sockaddr *)&csk->saddr);
+ }
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "hba %s, addr %s.\n", chba->ndev->name, buf);
+ break;
+ }
+ default:
+ return iscsi_host_get_param(shost, param, buf);
+ }
+
+ return len;
+}
+EXPORT_SYMBOL_GPL(cxgbi_get_host_param);
+
+struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
+ struct sockaddr *dst_addr,
+ int non_blocking)
+{
+ struct iscsi_endpoint *ep;
+ struct cxgbi_endpoint *cep;
+ struct cxgbi_hba *hba = NULL;
+ struct cxgbi_sock *csk;
+ int ifindex = 0;
+ int err = -EINVAL;
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
+ "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n",
+ shost, non_blocking, dst_addr);
+
+ if (shost) {
+ hba = iscsi_host_priv(shost);
+ if (!hba) {
+ pr_info("shost 0x%p, priv NULL.\n", shost);
+ goto err_out;
+ }
+ }
+
+check_route:
+ if (dst_addr->sa_family == AF_INET) {
+ csk = cxgbi_check_route(dst_addr, ifindex);
+#if IS_ENABLED(CONFIG_IPV6)
+ } else if (dst_addr->sa_family == AF_INET6) {
+ csk = cxgbi_check_route6(dst_addr, ifindex);
+#endif
+ } else {
+ pr_info("address family 0x%x NOT supported.\n",
+ dst_addr->sa_family);
+ err = -EAFNOSUPPORT;
+ return (struct iscsi_endpoint *)ERR_PTR(err);
+ }
+
+ if (IS_ERR(csk))
+ return (struct iscsi_endpoint *)csk;
+ cxgbi_sock_get(csk);
+
+ if (!hba)
+ hba = csk->cdev->hbas[csk->port_id];
+ else if (hba != csk->cdev->hbas[csk->port_id]) {
+ if (ifindex != hba->ndev->ifindex) {
+ cxgbi_sock_put(csk);
+ cxgbi_sock_closed(csk);
+ ifindex = hba->ndev->ifindex;
+ goto check_route;
+ }
+
+ pr_info("Could not connect through requested host %u"
+ "hba 0x%p != 0x%p (%u).\n",
+ shost->host_no, hba,
+ csk->cdev->hbas[csk->port_id], csk->port_id);
+ err = -ENOSPC;
+ goto release_conn;
+ }
+
+ err = sock_get_port(csk);
+ if (err)
+ goto release_conn;
+
+ cxgbi_sock_set_state(csk, CTP_CONNECTING);
+ err = csk->cdev->csk_init_act_open(csk);
+ if (err)
+ goto release_conn;
+
+ if (cxgbi_sock_is_closing(csk)) {
+ err = -ENOSPC;
+ pr_info("csk 0x%p is closing.\n", csk);
+ goto release_conn;
+ }
+
+ ep = iscsi_create_endpoint(sizeof(*cep));
+ if (!ep) {
+ err = -ENOMEM;
+ pr_info("iscsi alloc ep, OOM.\n");
+ goto release_conn;
+ }
+
+ cep = ep->dd_data;
+ cep->csk = csk;
+ cep->chba = hba;
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
+ "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n",
+ ep, cep, csk, hba, hba->ndev->name);
+ return ep;
+
+release_conn:
+ cxgbi_sock_put(csk);
+ cxgbi_sock_closed(csk);
+err_out:
+ return ERR_PTR(err);
+}
+EXPORT_SYMBOL_GPL(cxgbi_ep_connect);
+
+int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
+{
+ struct cxgbi_endpoint *cep = ep->dd_data;
+ struct cxgbi_sock *csk = cep->csk;
+
+ if (!cxgbi_sock_is_established(csk))
+ return 0;
+ return 1;
+}
+EXPORT_SYMBOL_GPL(cxgbi_ep_poll);
+
+void cxgbi_ep_disconnect(struct iscsi_endpoint *ep)
+{
+ struct cxgbi_endpoint *cep = ep->dd_data;
+ struct cxgbi_conn *cconn = cep->cconn;
+ struct cxgbi_sock *csk = cep->csk;
+
+ log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
+ "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n",
+ ep, cep, cconn, csk, csk->state, csk->flags);
+
+ if (cconn && cconn->iconn) {
+ iscsi_suspend_tx(cconn->iconn);
+ write_lock_bh(&csk->callback_lock);
+ cep->csk->user_data = NULL;
+ cconn->cep = NULL;
+ write_unlock_bh(&csk->callback_lock);
+ }
+ iscsi_destroy_endpoint(ep);
+
+ if (likely(csk->state >= CTP_ESTABLISHED))
+ need_active_close(csk);
+ else
+ cxgbi_sock_closed(csk);
+
+ cxgbi_sock_put(csk);
+}
+EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect);
+
+int cxgbi_iscsi_init(struct iscsi_transport *itp,
+ struct scsi_transport_template **stt)
+{
+ *stt = iscsi_register_transport(itp);
+ if (*stt == NULL) {
+ pr_err("unable to register %s transport 0x%p.\n",
+ itp->name, itp);
+ return -ENODEV;
+ }
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "%s, registered iscsi transport 0x%p.\n",
+ itp->name, stt);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(cxgbi_iscsi_init);
+
+void cxgbi_iscsi_cleanup(struct iscsi_transport *itp,
+ struct scsi_transport_template **stt)
+{
+ if (*stt) {
+ log_debug(1 << CXGBI_DBG_ISCSI,
+ "de-register transport 0x%p, %s, stt 0x%p.\n",
+ itp, itp->name, *stt);
+ *stt = NULL;
+ iscsi_unregister_transport(itp);
+ }
+}
+EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup);
+
+umode_t cxgbi_attr_is_visible(int param_type, int param)
+{
+ switch (param_type) {
+ case ISCSI_HOST_PARAM:
+ switch (param) {
+ case ISCSI_HOST_PARAM_NETDEV_NAME:
+ case ISCSI_HOST_PARAM_HWADDRESS:
+ case ISCSI_HOST_PARAM_IPADDRESS:
+ case ISCSI_HOST_PARAM_INITIATOR_NAME:
+ return S_IRUGO;
+ default:
+ return 0;
+ }
+ case ISCSI_PARAM:
+ switch (param) {
+ case ISCSI_PARAM_MAX_RECV_DLENGTH:
+ case ISCSI_PARAM_MAX_XMIT_DLENGTH:
+ case ISCSI_PARAM_HDRDGST_EN:
+ case ISCSI_PARAM_DATADGST_EN:
+ case ISCSI_PARAM_CONN_ADDRESS:
+ case ISCSI_PARAM_CONN_PORT:
+ case ISCSI_PARAM_EXP_STATSN:
+ case ISCSI_PARAM_PERSISTENT_ADDRESS:
+ case ISCSI_PARAM_PERSISTENT_PORT:
+ case ISCSI_PARAM_PING_TMO:
+ case ISCSI_PARAM_RECV_TMO:
+ case ISCSI_PARAM_INITIAL_R2T_EN:
+ case ISCSI_PARAM_MAX_R2T:
+ case ISCSI_PARAM_IMM_DATA_EN:
+ case ISCSI_PARAM_FIRST_BURST:
+ case ISCSI_PARAM_MAX_BURST:
+ case ISCSI_PARAM_PDU_INORDER_EN:
+ case ISCSI_PARAM_DATASEQ_INORDER_EN:
+ case ISCSI_PARAM_ERL:
+ case ISCSI_PARAM_TARGET_NAME:
+ case ISCSI_PARAM_TPGT:
+ case ISCSI_PARAM_USERNAME:
+ case ISCSI_PARAM_PASSWORD:
+ case ISCSI_PARAM_USERNAME_IN:
+ case ISCSI_PARAM_PASSWORD_IN:
+ case ISCSI_PARAM_FAST_ABORT:
+ case ISCSI_PARAM_ABORT_TMO:
+ case ISCSI_PARAM_LU_RESET_TMO:
+ case ISCSI_PARAM_TGT_RESET_TMO:
+ case ISCSI_PARAM_IFACE_NAME:
+ case ISCSI_PARAM_INITIATOR_NAME:
+ return S_IRUGO;
+ default:
+ return 0;
+ }
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible);
+
+static int __init libcxgbi_init_module(void)
+{
+ pr_info("%s", version);
+
+ BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) <
+ sizeof(struct cxgbi_skb_cb));
+ rsvd_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+ if (!rsvd_page)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void __exit libcxgbi_exit_module(void)
+{
+ cxgbi_device_unregister_all(0xFF);
+ put_page(rsvd_page);
+ return;
+}
+
+module_init(libcxgbi_init_module);
+module_exit(libcxgbi_exit_module);
diff --git a/drivers/scsi/cxgbi/libcxgbi.h b/drivers/scsi/cxgbi/libcxgbi.h
new file mode 100644
index 000000000..3687b5c0c
--- /dev/null
+++ b/drivers/scsi/cxgbi/libcxgbi.h
@@ -0,0 +1,647 @@
+/*
+ * libcxgbi.h: Chelsio common library for T3/T4 iSCSI driver.
+ *
+ * Copyright (c) 2010-2015 Chelsio Communications, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ *
+ * Written by: Karen Xie (kxie@chelsio.com)
+ * Written by: Rakesh Ranjan (rranjan@chelsio.com)
+ */
+
+#ifndef __LIBCXGBI_H__
+#define __LIBCXGBI_H__
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/debugfs.h>
+#include <linux/list.h>
+#include <linux/netdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/scatterlist.h>
+#include <linux/skbuff.h>
+#include <linux/vmalloc.h>
+#include <linux/version.h>
+#include <scsi/scsi_device.h>
+#include <scsi/libiscsi_tcp.h>
+
+#include <libcxgb_ppm.h>
+
+enum cxgbi_dbg_flag {
+ CXGBI_DBG_ISCSI,
+ CXGBI_DBG_DDP,
+ CXGBI_DBG_TOE,
+ CXGBI_DBG_SOCK,
+
+ CXGBI_DBG_PDU_TX,
+ CXGBI_DBG_PDU_RX,
+ CXGBI_DBG_DEV,
+};
+
+#define log_debug(level, fmt, ...) \
+ do { \
+ if (dbg_level & (level)) \
+ pr_info(fmt, ##__VA_ARGS__); \
+ } while (0)
+
+#define pr_info_ipaddr(fmt_trail, \
+ addr1, addr2, args_trail...) \
+do { \
+ if (!((1 << CXGBI_DBG_SOCK) & dbg_level)) \
+ break; \
+ pr_info("%pISpc - %pISpc, " fmt_trail, \
+ addr1, addr2, args_trail); \
+} while (0)
+
+/* max. connections per adapter */
+#define CXGBI_MAX_CONN 16384
+
+/* always allocate rooms for AHS */
+#define SKB_TX_ISCSI_PDU_HEADER_MAX \
+ (sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE)
+
+#define ISCSI_PDU_NONPAYLOAD_LEN 312 /* bhs(48) + ahs(256) + digest(8)*/
+
+/*
+ * align pdu size to multiple of 512 for better performance
+ */
+#define cxgbi_align_pdu_size(n) do { n = (n) & (~511); } while (0)
+
+#define ULP2_MODE_ISCSI 2
+
+#define ULP2_MAX_PKT_SIZE 16224
+#define ULP2_MAX_PDU_PAYLOAD \
+ (ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_LEN)
+
+#define CXGBI_ULP2_MAX_ISO_PAYLOAD 65535
+
+#define CXGBI_MAX_ISO_DATA_IN_SKB \
+ min_t(u32, MAX_SKB_FRAGS << PAGE_SHIFT, CXGBI_ULP2_MAX_ISO_PAYLOAD)
+
+#define cxgbi_is_iso_config(csk) ((csk)->cdev->skb_iso_txhdr)
+#define cxgbi_is_iso_disabled(csk) ((csk)->disable_iso)
+
+/*
+ * For iscsi connections HW may inserts digest bytes into the pdu. Those digest
+ * bytes are not sent by the host but are part of the TCP payload and therefore
+ * consume TCP sequence space.
+ */
+static const unsigned int ulp2_extra_len[] = { 0, 4, 4, 8 };
+static inline unsigned int cxgbi_ulp_extra_len(int submode)
+{
+ return ulp2_extra_len[submode & 3];
+}
+
+#define CPL_RX_DDP_STATUS_DDP_SHIFT 16 /* ddp'able */
+#define CPL_RX_DDP_STATUS_PAD_SHIFT 19 /* pad error */
+#define CPL_RX_DDP_STATUS_HCRC_SHIFT 20 /* hcrc error */
+#define CPL_RX_DDP_STATUS_DCRC_SHIFT 21 /* dcrc error */
+
+/*
+ * sge_opaque_hdr -
+ * Opaque version of structure the SGE stores at skb->head of TX_DATA packets
+ * and for which we must reserve space.
+ */
+struct sge_opaque_hdr {
+ void *dev;
+ dma_addr_t addr[MAX_SKB_FRAGS + 1];
+};
+
+struct cxgbi_sock {
+ struct cxgbi_device *cdev;
+
+ int tid;
+ int atid;
+ unsigned long flags;
+ unsigned int mtu;
+ unsigned short rss_qid;
+ unsigned short txq_idx;
+ unsigned short advmss;
+ unsigned int tx_chan;
+ unsigned int rx_chan;
+ unsigned int mss_idx;
+ unsigned int smac_idx;
+ unsigned char port_id;
+ int wr_max_cred;
+ int wr_cred;
+ int wr_una_cred;
+#ifdef CONFIG_CHELSIO_T4_DCB
+ u8 dcb_priority;
+#endif
+ unsigned char hcrc_len;
+ unsigned char dcrc_len;
+
+ void *l2t;
+ struct sk_buff *wr_pending_head;
+ struct sk_buff *wr_pending_tail;
+ struct sk_buff *cpl_close;
+ struct sk_buff *cpl_abort_req;
+ struct sk_buff *cpl_abort_rpl;
+ struct sk_buff *skb_ulp_lhdr;
+ spinlock_t lock;
+ struct kref refcnt;
+ unsigned int state;
+ unsigned int csk_family;
+ union {
+ struct sockaddr_in saddr;
+ struct sockaddr_in6 saddr6;
+ };
+ union {
+ struct sockaddr_in daddr;
+ struct sockaddr_in6 daddr6;
+ };
+ struct dst_entry *dst;
+ struct sk_buff_head receive_queue;
+ struct sk_buff_head write_queue;
+ struct timer_list retry_timer;
+ struct completion cmpl;
+ int err;
+ rwlock_t callback_lock;
+ void *user_data;
+
+ u32 rcv_nxt;
+ u32 copied_seq;
+ u32 rcv_wup;
+ u32 snd_nxt;
+ u32 snd_una;
+ u32 write_seq;
+ u32 snd_win;
+ u32 rcv_win;
+
+ bool disable_iso;
+ u32 no_tx_credits;
+ unsigned long prev_iso_ts;
+};
+
+/*
+ * connection states
+ */
+enum cxgbi_sock_states{
+ CTP_CLOSED,
+ CTP_CONNECTING,
+ CTP_ACTIVE_OPEN,
+ CTP_ESTABLISHED,
+ CTP_ACTIVE_CLOSE,
+ CTP_PASSIVE_CLOSE,
+ CTP_CLOSE_WAIT_1,
+ CTP_CLOSE_WAIT_2,
+ CTP_ABORTING,
+};
+
+/*
+ * Connection flags -- many to track some close related events.
+ */
+enum cxgbi_sock_flags {
+ CTPF_ABORT_RPL_RCVD, /*received one ABORT_RPL_RSS message */
+ CTPF_ABORT_REQ_RCVD, /*received one ABORT_REQ_RSS message */
+ CTPF_ABORT_RPL_PENDING, /* expecting an abort reply */
+ CTPF_TX_DATA_SENT, /* already sent a TX_DATA WR */
+ CTPF_ACTIVE_CLOSE_NEEDED,/* need to be closed */
+ CTPF_HAS_ATID, /* reserved atid */
+ CTPF_HAS_TID, /* reserved hw tid */
+ CTPF_OFFLOAD_DOWN, /* offload function off */
+ CTPF_LOGOUT_RSP_RCVD, /* received logout response */
+};
+
+struct cxgbi_skb_rx_cb {
+ __u32 ddigest;
+ __u32 pdulen;
+};
+
+struct cxgbi_skb_tx_cb {
+ void *handle;
+ void *arp_err_handler;
+ struct sk_buff *wr_next;
+ u16 iscsi_hdr_len;
+ u8 ulp_mode;
+};
+
+enum cxgbi_skcb_flags {
+ SKCBF_TX_NEED_HDR, /* packet needs a header */
+ SKCBF_TX_MEM_WRITE, /* memory write */
+ SKCBF_TX_FLAG_COMPL, /* wr completion flag */
+ SKCBF_RX_COALESCED, /* received whole pdu */
+ SKCBF_RX_HDR, /* received pdu header */
+ SKCBF_RX_DATA, /* received pdu payload */
+ SKCBF_RX_STATUS, /* received ddp status */
+ SKCBF_RX_ISCSI_COMPL, /* received iscsi completion */
+ SKCBF_RX_DATA_DDPD, /* pdu payload ddp'd */
+ SKCBF_RX_HCRC_ERR, /* header digest error */
+ SKCBF_RX_DCRC_ERR, /* data digest error */
+ SKCBF_RX_PAD_ERR, /* padding byte error */
+ SKCBF_TX_ISO, /* iso cpl in tx skb */
+};
+
+struct cxgbi_skb_cb {
+ union {
+ struct cxgbi_skb_rx_cb rx;
+ struct cxgbi_skb_tx_cb tx;
+ };
+ unsigned long flags;
+ unsigned int seq;
+};
+
+#define CXGBI_SKB_CB(skb) ((struct cxgbi_skb_cb *)&((skb)->cb[0]))
+#define cxgbi_skcb_flags(skb) (CXGBI_SKB_CB(skb)->flags)
+#define cxgbi_skcb_tcp_seq(skb) (CXGBI_SKB_CB(skb)->seq)
+#define cxgbi_skcb_rx_ddigest(skb) (CXGBI_SKB_CB(skb)->rx.ddigest)
+#define cxgbi_skcb_rx_pdulen(skb) (CXGBI_SKB_CB(skb)->rx.pdulen)
+#define cxgbi_skcb_tx_wr_next(skb) (CXGBI_SKB_CB(skb)->tx.wr_next)
+#define cxgbi_skcb_tx_iscsi_hdrlen(skb) (CXGBI_SKB_CB(skb)->tx.iscsi_hdr_len)
+#define cxgbi_skcb_tx_ulp_mode(skb) (CXGBI_SKB_CB(skb)->tx.ulp_mode)
+
+static inline void cxgbi_skcb_set_flag(struct sk_buff *skb,
+ enum cxgbi_skcb_flags flag)
+{
+ __set_bit(flag, &(cxgbi_skcb_flags(skb)));
+}
+
+static inline void cxgbi_skcb_clear_flag(struct sk_buff *skb,
+ enum cxgbi_skcb_flags flag)
+{
+ __clear_bit(flag, &(cxgbi_skcb_flags(skb)));
+}
+
+static inline int cxgbi_skcb_test_flag(const struct sk_buff *skb,
+ enum cxgbi_skcb_flags flag)
+{
+ return test_bit(flag, &(cxgbi_skcb_flags(skb)));
+}
+
+static inline void cxgbi_sock_set_flag(struct cxgbi_sock *csk,
+ enum cxgbi_sock_flags flag)
+{
+ __set_bit(flag, &csk->flags);
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx, bit %d.\n",
+ csk, csk->state, csk->flags, flag);
+}
+
+static inline void cxgbi_sock_clear_flag(struct cxgbi_sock *csk,
+ enum cxgbi_sock_flags flag)
+{
+ __clear_bit(flag, &csk->flags);
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx, bit %d.\n",
+ csk, csk->state, csk->flags, flag);
+}
+
+static inline int cxgbi_sock_flag(struct cxgbi_sock *csk,
+ enum cxgbi_sock_flags flag)
+{
+ if (csk == NULL)
+ return 0;
+ return test_bit(flag, &csk->flags);
+}
+
+static inline void cxgbi_sock_set_state(struct cxgbi_sock *csk, int state)
+{
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "csk 0x%p,%u,0x%lx, state -> %u.\n",
+ csk, csk->state, csk->flags, state);
+ csk->state = state;
+}
+
+static inline void cxgbi_sock_free(struct kref *kref)
+{
+ struct cxgbi_sock *csk = container_of(kref,
+ struct cxgbi_sock,
+ refcnt);
+ if (csk) {
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "free csk 0x%p, state %u, flags 0x%lx\n",
+ csk, csk->state, csk->flags);
+ kfree(csk);
+ }
+}
+
+static inline void __cxgbi_sock_put(const char *fn, struct cxgbi_sock *csk)
+{
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "%s, put csk 0x%p, ref %u-1.\n",
+ fn, csk, kref_read(&csk->refcnt));
+ kref_put(&csk->refcnt, cxgbi_sock_free);
+}
+#define cxgbi_sock_put(csk) __cxgbi_sock_put(__func__, csk)
+
+static inline void __cxgbi_sock_get(const char *fn, struct cxgbi_sock *csk)
+{
+ log_debug(1 << CXGBI_DBG_SOCK,
+ "%s, get csk 0x%p, ref %u+1.\n",
+ fn, csk, kref_read(&csk->refcnt));
+ kref_get(&csk->refcnt);
+}
+#define cxgbi_sock_get(csk) __cxgbi_sock_get(__func__, csk)
+
+static inline int cxgbi_sock_is_closing(struct cxgbi_sock *csk)
+{
+ return csk->state >= CTP_ACTIVE_CLOSE;
+}
+
+static inline int cxgbi_sock_is_established(struct cxgbi_sock *csk)
+{
+ return csk->state == CTP_ESTABLISHED;
+}
+
+static inline void cxgbi_sock_purge_write_queue(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb;
+
+ while ((skb = __skb_dequeue(&csk->write_queue)))
+ __kfree_skb(skb);
+}
+
+static inline unsigned int cxgbi_sock_compute_wscale(unsigned int win)
+{
+ unsigned int wscale = 0;
+
+ while (wscale < 14 && (65535 << wscale) < win)
+ wscale++;
+ return wscale;
+}
+
+static inline struct sk_buff *alloc_wr(int wrlen, int dlen, gfp_t gfp)
+{
+ struct sk_buff *skb = alloc_skb(wrlen + dlen, gfp);
+
+ if (skb) {
+ __skb_put(skb, wrlen);
+ memset(skb->head, 0, wrlen + dlen);
+ } else
+ pr_info("alloc cpl wr skb %u+%u, OOM.\n", wrlen, dlen);
+ return skb;
+}
+
+
+/*
+ * The number of WRs needed for an skb depends on the number of fragments
+ * in the skb and whether it has any payload in its main body. This maps the
+ * length of the gather list represented by an skb into the # of necessary WRs.
+ * The extra two fragments are for iscsi bhs and payload padding.
+ */
+#define SKB_WR_LIST_SIZE (MAX_SKB_FRAGS + 2)
+
+static inline void cxgbi_sock_reset_wr_list(struct cxgbi_sock *csk)
+{
+ csk->wr_pending_head = csk->wr_pending_tail = NULL;
+}
+
+static inline void cxgbi_sock_enqueue_wr(struct cxgbi_sock *csk,
+ struct sk_buff *skb)
+{
+ cxgbi_skcb_tx_wr_next(skb) = NULL;
+ /*
+ * We want to take an extra reference since both us and the driver
+ * need to free the packet before it's really freed.
+ */
+ skb_get(skb);
+
+ if (!csk->wr_pending_head)
+ csk->wr_pending_head = skb;
+ else
+ cxgbi_skcb_tx_wr_next(csk->wr_pending_tail) = skb;
+ csk->wr_pending_tail = skb;
+}
+
+static inline int cxgbi_sock_count_pending_wrs(const struct cxgbi_sock *csk)
+{
+ int n = 0;
+ const struct sk_buff *skb = csk->wr_pending_head;
+
+ while (skb) {
+ n += skb->csum;
+ skb = cxgbi_skcb_tx_wr_next(skb);
+ }
+ return n;
+}
+
+static inline struct sk_buff *cxgbi_sock_peek_wr(const struct cxgbi_sock *csk)
+{
+ return csk->wr_pending_head;
+}
+
+static inline struct sk_buff *cxgbi_sock_dequeue_wr(struct cxgbi_sock *csk)
+{
+ struct sk_buff *skb = csk->wr_pending_head;
+
+ if (likely(skb)) {
+ csk->wr_pending_head = cxgbi_skcb_tx_wr_next(skb);
+ cxgbi_skcb_tx_wr_next(skb) = NULL;
+ }
+ return skb;
+}
+
+void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *);
+void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *);
+void cxgbi_sock_skb_entail(struct cxgbi_sock *, struct sk_buff *);
+void cxgbi_sock_fail_act_open(struct cxgbi_sock *, int);
+void cxgbi_sock_act_open_req_arp_failure(void *, struct sk_buff *);
+void cxgbi_sock_closed(struct cxgbi_sock *);
+void cxgbi_sock_established(struct cxgbi_sock *, unsigned int, unsigned int);
+void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *);
+void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *);
+void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *, u32);
+void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *, unsigned int, unsigned int,
+ int);
+unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *, unsigned int);
+void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *);
+
+struct cxgbi_hba {
+ struct net_device *ndev;
+ struct net_device *vdev; /* vlan dev */
+ struct Scsi_Host *shost;
+ struct cxgbi_device *cdev;
+ __be32 ipv4addr;
+ unsigned char port_id;
+};
+
+struct cxgbi_ports_map {
+ unsigned int max_connect;
+ unsigned int used;
+ unsigned short sport_base;
+ spinlock_t lock;
+ unsigned int next;
+ struct cxgbi_sock **port_csk;
+};
+
+#define CXGBI_FLAG_DEV_T3 0x1
+#define CXGBI_FLAG_DEV_T4 0x2
+#define CXGBI_FLAG_ADAPTER_RESET 0x4
+#define CXGBI_FLAG_IPV4_SET 0x10
+#define CXGBI_FLAG_USE_PPOD_OFLDQ 0x40
+#define CXGBI_FLAG_DDP_OFF 0x100
+#define CXGBI_FLAG_DEV_ISO_OFF 0x400
+
+struct cxgbi_device {
+ struct list_head list_head;
+ struct list_head rcu_node;
+ unsigned int flags;
+ struct net_device **ports;
+ void *lldev;
+ struct cxgbi_hba **hbas;
+ const unsigned short *mtus;
+ unsigned char nmtus;
+ unsigned char nports;
+ struct pci_dev *pdev;
+ struct dentry *debugfs_root;
+ struct iscsi_transport *itp;
+ struct module *owner;
+
+ unsigned int pfvf;
+ unsigned int rx_credit_thres;
+ unsigned int skb_tx_rsvd;
+ u32 skb_iso_txhdr;
+ unsigned int skb_rx_extra; /* for msg coalesced mode */
+ unsigned int tx_max_size;
+ unsigned int rx_max_size;
+ unsigned int rxq_idx_cntr;
+ struct cxgbi_ports_map pmap;
+
+ void (*dev_ddp_cleanup)(struct cxgbi_device *);
+ struct cxgbi_ppm* (*cdev2ppm)(struct cxgbi_device *);
+ int (*csk_ddp_set_map)(struct cxgbi_ppm *, struct cxgbi_sock *,
+ struct cxgbi_task_tag_info *);
+ void (*csk_ddp_clear_map)(struct cxgbi_device *cdev,
+ struct cxgbi_ppm *,
+ struct cxgbi_task_tag_info *);
+ int (*csk_ddp_setup_digest)(struct cxgbi_sock *,
+ unsigned int, int, int);
+ int (*csk_ddp_setup_pgidx)(struct cxgbi_sock *,
+ unsigned int, int);
+
+ void (*csk_release_offload_resources)(struct cxgbi_sock *);
+ int (*csk_rx_pdu_ready)(struct cxgbi_sock *, struct sk_buff *);
+ u32 (*csk_send_rx_credits)(struct cxgbi_sock *, u32);
+ int (*csk_push_tx_frames)(struct cxgbi_sock *, int);
+ void (*csk_send_abort_req)(struct cxgbi_sock *);
+ void (*csk_send_close_req)(struct cxgbi_sock *);
+ int (*csk_alloc_cpls)(struct cxgbi_sock *);
+ int (*csk_init_act_open)(struct cxgbi_sock *);
+
+ void *dd_data;
+};
+#define cxgbi_cdev_priv(cdev) ((cdev)->dd_data)
+
+struct cxgbi_conn {
+ struct cxgbi_endpoint *cep;
+ struct iscsi_conn *iconn;
+ struct cxgbi_hba *chba;
+ u32 task_idx_bits;
+ unsigned int ddp_full;
+ unsigned int ddp_tag_full;
+};
+
+struct cxgbi_endpoint {
+ struct cxgbi_conn *cconn;
+ struct cxgbi_hba *chba;
+ struct cxgbi_sock *csk;
+};
+
+struct cxgbi_task_data {
+#define CXGBI_TASK_SGL_CHECKED 0x1
+#define CXGBI_TASK_SGL_COPY 0x2
+ u8 flags;
+ unsigned short nr_frags;
+ struct page_frag frags[MAX_SKB_FRAGS];
+ struct sk_buff *skb;
+ unsigned int dlen;
+ unsigned int offset;
+ unsigned int count;
+ unsigned int sgoffset;
+ u32 total_count;
+ u32 total_offset;
+ u32 max_xmit_dlength;
+ struct cxgbi_task_tag_info ttinfo;
+};
+#define iscsi_task_cxgbi_data(task) \
+ ((task)->dd_data + sizeof(struct iscsi_tcp_task))
+
+struct cxgbi_iso_info {
+#define CXGBI_ISO_INFO_FSLICE 0x1
+#define CXGBI_ISO_INFO_LSLICE 0x2
+#define CXGBI_ISO_INFO_IMM_ENABLE 0x4
+ u8 flags;
+ u8 op;
+ u8 ahs;
+ u8 num_pdu;
+ u32 mpdu;
+ u32 burst_size;
+ u32 len;
+ u32 segment_offset;
+ u32 datasn_offset;
+ u32 buffer_offset;
+};
+
+static inline void cxgbi_set_iscsi_ipv4(struct cxgbi_hba *chba, __be32 ipaddr)
+{
+ if (chba->cdev->flags & CXGBI_FLAG_IPV4_SET)
+ chba->ipv4addr = ipaddr;
+ else
+ pr_info("set iscsi ipv4 NOT supported, using %s ipv4.\n",
+ chba->ndev->name);
+}
+
+struct cxgbi_device *cxgbi_device_register(unsigned int, unsigned int);
+void cxgbi_device_unregister(struct cxgbi_device *);
+void cxgbi_device_unregister_all(unsigned int flag);
+struct cxgbi_device *cxgbi_device_find_by_lldev(void *);
+struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *, int *);
+struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *,
+ int *);
+int cxgbi_hbas_add(struct cxgbi_device *, u64, unsigned int,
+ struct scsi_host_template *,
+ struct scsi_transport_template *);
+void cxgbi_hbas_remove(struct cxgbi_device *);
+
+int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base,
+ unsigned int max_conn);
+void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev);
+
+void cxgbi_conn_tx_open(struct cxgbi_sock *);
+void cxgbi_conn_pdu_ready(struct cxgbi_sock *);
+int cxgbi_conn_alloc_pdu(struct iscsi_task *, u8);
+int cxgbi_conn_init_pdu(struct iscsi_task *, unsigned int , unsigned int);
+int cxgbi_conn_xmit_pdu(struct iscsi_task *);
+
+void cxgbi_cleanup_task(struct iscsi_task *task);
+
+umode_t cxgbi_attr_is_visible(int param_type, int param);
+void cxgbi_get_conn_stats(struct iscsi_cls_conn *, struct iscsi_stats *);
+int cxgbi_set_conn_param(struct iscsi_cls_conn *,
+ enum iscsi_param, char *, int);
+int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param, char *);
+struct iscsi_cls_conn *cxgbi_create_conn(struct iscsi_cls_session *, u32);
+int cxgbi_bind_conn(struct iscsi_cls_session *,
+ struct iscsi_cls_conn *, u64, int);
+void cxgbi_destroy_session(struct iscsi_cls_session *);
+struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *,
+ u16, u16, u32);
+int cxgbi_set_host_param(struct Scsi_Host *,
+ enum iscsi_host_param, char *, int);
+int cxgbi_get_host_param(struct Scsi_Host *, enum iscsi_host_param, char *);
+struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *,
+ struct sockaddr *, int);
+int cxgbi_ep_poll(struct iscsi_endpoint *, int);
+void cxgbi_ep_disconnect(struct iscsi_endpoint *);
+
+int cxgbi_iscsi_init(struct iscsi_transport *,
+ struct scsi_transport_template **);
+void cxgbi_iscsi_cleanup(struct iscsi_transport *,
+ struct scsi_transport_template **);
+void cxgbi_parse_pdu_itt(struct iscsi_conn *, itt_t, int *, int *);
+int cxgbi_ddp_init(struct cxgbi_device *, unsigned int, unsigned int,
+ unsigned int, unsigned int);
+int cxgbi_ddp_cleanup(struct cxgbi_device *);
+void cxgbi_ddp_page_size_factor(int *);
+void cxgbi_ddp_set_one_ppod(struct cxgbi_pagepod *,
+ struct cxgbi_task_tag_info *,
+ struct scatterlist **sg_pp, unsigned int *sg_off);
+int cxgbi_ddp_ppm_setup(void **ppm_pp, struct cxgbi_device *cdev,
+ struct cxgbi_tag_format *tformat,
+ unsigned int iscsi_size, unsigned int llimit,
+ unsigned int start, unsigned int rsvd_factor,
+ unsigned int edram_start, unsigned int edram_size);
+#endif /*__LIBCXGBI_H__*/
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-16 00:22:30 +0000