diff options
Diffstat (limited to 'drivers/scsi/fnic')
| -rw-r--r-- | drivers/scsi/fnic/fnic.h | 71 | ||||
| -rw-r--r-- | drivers/scsi/fnic/fnic_fcs.c | 68 | ||||
| -rw-r--r-- | drivers/scsi/fnic/fnic_isr.c | 168 | ||||
| -rw-r--r-- | drivers/scsi/fnic/fnic_main.c | 145 | ||||
| -rw-r--r-- | drivers/scsi/fnic/fnic_res.c | 48 | ||||
| -rw-r--r-- | drivers/scsi/fnic/fnic_scsi.c | 873 | ||||
| -rw-r--r-- | drivers/scsi/fnic/fnic_stats.h | 3 | ||||
| -rw-r--r-- | drivers/scsi/fnic/fnic_trace.c | 11 | ||||
| -rw-r--r-- | drivers/scsi/fnic/vnic_dev.c | 4 | ||||
| -rw-r--r-- | drivers/scsi/fnic/vnic_scsi.h | 13 |
10 files changed, 856 insertions, 548 deletions
diff --git a/drivers/scsi/fnic/fnic.h b/drivers/scsi/fnic/fnic.h index 22cef283b2..ce73f08ee8 100644 --- a/drivers/scsi/fnic/fnic.h +++ b/drivers/scsi/fnic/fnic.h @@ -27,7 +27,7 @@ #define DRV_NAME "fnic" #define DRV_DESCRIPTION "Cisco FCoE HBA Driver" -#define DRV_VERSION "1.6.0.57" +#define DRV_VERSION "1.7.0.0" #define PFX DRV_NAME ": " #define DFX DRV_NAME "%d: " @@ -36,7 +36,6 @@ #define FNIC_MIN_IO_REQ 256 /* Min IO throttle count */ #define FNIC_MAX_IO_REQ 1024 /* scsi_cmnd tag map entries */ #define FNIC_DFLT_IO_REQ 256 /* Default scsi_cmnd tag map entries */ -#define FNIC_IO_LOCKS 64 /* IO locks: power of 2 */ #define FNIC_DFLT_QUEUE_DEPTH 256 #define FNIC_STATS_RATE_LIMIT 4 /* limit rate at which stats are pulled up */ @@ -109,7 +108,7 @@ static inline u64 fnic_flags_and_state(struct scsi_cmnd *cmd) #define FNIC_ABT_TERM_DELAY_TIMEOUT 500 /* mSec */ #define FNIC_MAX_FCP_TARGET 256 - +#define FNIC_PCI_OFFSET 2 /** * state_flags to identify host state along along with fnic's state **/ @@ -144,31 +143,48 @@ do { \ } while (0); \ } while (0) -#define FNIC_MAIN_DBG(kern_level, host, fmt, args...) \ +#define FNIC_MAIN_DBG(kern_level, host, fnic_num, fmt, args...) \ FNIC_CHECK_LOGGING(FNIC_MAIN_LOGGING, \ - shost_printk(kern_level, host, fmt, ##args);) + shost_printk(kern_level, host, \ + "fnic<%d>: %s: %d: " fmt, fnic_num,\ + __func__, __LINE__, ##args);) -#define FNIC_FCS_DBG(kern_level, host, fmt, args...) \ +#define FNIC_FCS_DBG(kern_level, host, fnic_num, fmt, args...) \ FNIC_CHECK_LOGGING(FNIC_FCS_LOGGING, \ - shost_printk(kern_level, host, fmt, ##args);) + shost_printk(kern_level, host, \ + "fnic<%d>: %s: %d: " fmt, fnic_num,\ + __func__, __LINE__, ##args);) -#define FNIC_SCSI_DBG(kern_level, host, fmt, args...) \ +#define FNIC_SCSI_DBG(kern_level, host, fnic_num, fmt, args...) \ FNIC_CHECK_LOGGING(FNIC_SCSI_LOGGING, \ - shost_printk(kern_level, host, fmt, ##args);) + shost_printk(kern_level, host, \ + "fnic<%d>: %s: %d: " fmt, fnic_num,\ + __func__, __LINE__, ##args);) -#define FNIC_ISR_DBG(kern_level, host, fmt, args...) \ +#define FNIC_ISR_DBG(kern_level, host, fnic_num, fmt, args...) \ FNIC_CHECK_LOGGING(FNIC_ISR_LOGGING, \ - shost_printk(kern_level, host, fmt, ##args);) + shost_printk(kern_level, host, \ + "fnic<%d>: %s: %d: " fmt, fnic_num,\ + __func__, __LINE__, ##args);) #define FNIC_MAIN_NOTE(kern_level, host, fmt, args...) \ shost_printk(kern_level, host, fmt, ##args) +#define FNIC_WQ_COPY_MAX 64 +#define FNIC_WQ_MAX 1 +#define FNIC_RQ_MAX 1 +#define FNIC_CQ_MAX (FNIC_WQ_COPY_MAX + FNIC_WQ_MAX + FNIC_RQ_MAX) +#define FNIC_DFLT_IO_COMPLETIONS 256 + +#define FNIC_MQ_CQ_INDEX 2 + extern const char *fnic_state_str[]; enum fnic_intx_intr_index { FNIC_INTX_WQ_RQ_COPYWQ, - FNIC_INTX_ERR, + FNIC_INTX_DUMMY, FNIC_INTX_NOTIFY, + FNIC_INTX_ERR, FNIC_INTX_INTR_MAX, }; @@ -176,7 +192,7 @@ enum fnic_msix_intr_index { FNIC_MSIX_RQ, FNIC_MSIX_WQ, FNIC_MSIX_WQ_COPY, - FNIC_MSIX_ERR_NOTIFY, + FNIC_MSIX_ERR_NOTIFY = FNIC_MSIX_WQ_COPY + FNIC_WQ_COPY_MAX, FNIC_MSIX_INTR_MAX, }; @@ -185,6 +201,7 @@ struct fnic_msix_entry { char devname[IFNAMSIZ + 11]; irqreturn_t (*isr)(int, void *); void *devid; + int irq_num; }; enum fnic_state { @@ -194,12 +211,6 @@ enum fnic_state { FNIC_IN_ETH_TRANS_FC_MODE, }; -#define FNIC_WQ_COPY_MAX 1 -#define FNIC_WQ_MAX 1 -#define FNIC_RQ_MAX 1 -#define FNIC_CQ_MAX (FNIC_WQ_COPY_MAX + FNIC_WQ_MAX + FNIC_RQ_MAX) -#define FNIC_DFLT_IO_COMPLETIONS 256 - struct mempool; enum fnic_evt { @@ -214,8 +225,16 @@ struct fnic_event { enum fnic_evt event; }; +struct fnic_cpy_wq { + unsigned long hw_lock_flags; + u16 active_ioreq_count; + u16 ioreq_table_size; + ____cacheline_aligned struct fnic_io_req **io_req_table; +}; + /* Per-instance private data structure */ struct fnic { + int fnic_num; struct fc_lport *lport; struct fcoe_ctlr ctlr; /* FIP FCoE controller structure */ struct vnic_dev_bar bar0; @@ -282,10 +301,11 @@ struct fnic { struct fnic_host_tag *tags; mempool_t *io_req_pool; mempool_t *io_sgl_pool[FNIC_SGL_NUM_CACHES]; - spinlock_t io_req_lock[FNIC_IO_LOCKS]; /* locks for scsi cmnds */ + unsigned int copy_wq_base; struct work_struct link_work; struct work_struct frame_work; + struct work_struct flush_work; struct sk_buff_head frame_queue; struct sk_buff_head tx_queue; @@ -302,7 +322,9 @@ struct fnic { /*** FIP related data members -- end ***/ /* copy work queue cache line section */ - ____cacheline_aligned struct vnic_wq_copy wq_copy[FNIC_WQ_COPY_MAX]; + ____cacheline_aligned struct vnic_wq_copy hw_copy_wq[FNIC_WQ_COPY_MAX]; + ____cacheline_aligned struct fnic_cpy_wq sw_copy_wq[FNIC_WQ_COPY_MAX]; + /* completion queue cache line section */ ____cacheline_aligned struct vnic_cq cq[FNIC_CQ_MAX]; @@ -330,6 +352,7 @@ extern const struct attribute_group *fnic_host_groups[]; void fnic_clear_intr_mode(struct fnic *fnic); int fnic_set_intr_mode(struct fnic *fnic); +int fnic_set_intr_mode_msix(struct fnic *fnic); void fnic_free_intr(struct fnic *fnic); int fnic_request_intr(struct fnic *fnic); @@ -341,7 +364,7 @@ void fnic_handle_event(struct work_struct *work); int fnic_rq_cmpl_handler(struct fnic *fnic, int); int fnic_alloc_rq_frame(struct vnic_rq *rq); void fnic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf); -void fnic_flush_tx(struct fnic *); +void fnic_flush_tx(struct work_struct *work); void fnic_eth_send(struct fcoe_ctlr *, struct sk_buff *skb); void fnic_set_port_id(struct fc_lport *, u32, struct fc_frame *); void fnic_update_mac(struct fc_lport *, u8 *new); @@ -356,7 +379,7 @@ void fnic_scsi_cleanup(struct fc_lport *); void fnic_scsi_abort_io(struct fc_lport *); void fnic_empty_scsi_cleanup(struct fc_lport *); void fnic_exch_mgr_reset(struct fc_lport *, u32, u32); -int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int); +int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do, unsigned int cq_index); int fnic_wq_cmpl_handler(struct fnic *fnic, int); int fnic_flogi_reg_handler(struct fnic *fnic, u32); void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, @@ -364,7 +387,7 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, int fnic_fw_reset_handler(struct fnic *fnic); void fnic_terminate_rport_io(struct fc_rport *); const char *fnic_state_to_str(unsigned int state); - +void fnic_mq_map_queues_cpus(struct Scsi_Host *host); void fnic_log_q_error(struct fnic *fnic); void fnic_handle_link_event(struct fnic *fnic); diff --git a/drivers/scsi/fnic/fnic_fcs.c b/drivers/scsi/fnic/fnic_fcs.c index 55632c67a8..a08293b2ad 100644 --- a/drivers/scsi/fnic/fnic_fcs.c +++ b/drivers/scsi/fnic/fnic_fcs.c @@ -63,8 +63,8 @@ void fnic_handle_link(struct work_struct *work) atomic64_set(&fnic->fnic_stats.misc_stats.current_port_speed, new_port_speed); if (old_port_speed != new_port_speed) - FNIC_MAIN_DBG(KERN_INFO, fnic->lport->host, - "Current vnic speed set to : %llu\n", + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "Current vnic speed set to: %llu\n", new_port_speed); switch (vnic_dev_port_speed(fnic->vdev)) { @@ -102,6 +102,8 @@ void fnic_handle_link(struct work_struct *work) fnic_fc_trace_set_data(fnic->lport->host->host_no, FNIC_FC_LE, "Link Status: DOWN->DOWN", strlen("Link Status: DOWN->DOWN")); + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "down->down\n"); } else { if (old_link_down_cnt != fnic->link_down_cnt) { /* UP -> DOWN -> UP */ @@ -113,7 +115,7 @@ void fnic_handle_link(struct work_struct *work) "Link Status:UP_DOWN_UP", strlen("Link_Status:UP_DOWN_UP") ); - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "link down\n"); fcoe_ctlr_link_down(&fnic->ctlr); if (fnic->config.flags & VFCF_FIP_CAPABLE) { @@ -128,8 +130,8 @@ void fnic_handle_link(struct work_struct *work) fnic_fcoe_send_vlan_req(fnic); return; } - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, - "link up\n"); + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "up->down->up: Link up\n"); fcoe_ctlr_link_up(&fnic->ctlr); } else { /* UP -> UP */ @@ -138,6 +140,8 @@ void fnic_handle_link(struct work_struct *work) fnic->lport->host->host_no, FNIC_FC_LE, "Link Status: UP_UP", strlen("Link Status: UP_UP")); + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "up->up\n"); } } } else if (fnic->link_status) { @@ -153,7 +157,8 @@ void fnic_handle_link(struct work_struct *work) return; } - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, "link up\n"); + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "down->up: Link up\n"); fnic_fc_trace_set_data(fnic->lport->host->host_no, FNIC_FC_LE, "Link Status: DOWN_UP", strlen("Link Status: DOWN_UP")); fcoe_ctlr_link_up(&fnic->ctlr); @@ -161,13 +166,14 @@ void fnic_handle_link(struct work_struct *work) /* UP -> DOWN */ fnic->lport->host_stats.link_failure_count++; spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, "link down\n"); + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "up->down: Link down\n"); fnic_fc_trace_set_data( fnic->lport->host->host_no, FNIC_FC_LE, "Link Status: UP_DOWN", strlen("Link Status: UP_DOWN")); if (fnic->config.flags & VFCF_FIP_CAPABLE) { - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "deleting fip-timer during link-down\n"); del_timer_sync(&fnic->fip_timer); } @@ -270,12 +276,12 @@ void fnic_handle_event(struct work_struct *work) spin_lock_irqsave(&fnic->fnic_lock, flags); break; case FNIC_EVT_START_FCF_DISC: - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Start FCF Discovery\n"); fnic_fcoe_start_fcf_disc(fnic); break; default: - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Unknown event 0x%x\n", fevt->event); break; } @@ -370,7 +376,7 @@ static void fnic_fcoe_send_vlan_req(struct fnic *fnic) fnic->set_vlan(fnic, 0); if (printk_ratelimit()) - FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "Sending VLAN request...\n"); skb = dev_alloc_skb(sizeof(struct fip_vlan)); @@ -423,12 +429,12 @@ static void fnic_fcoe_process_vlan_resp(struct fnic *fnic, struct sk_buff *skb) u64 sol_time; unsigned long flags; - FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "Received VLAN response...\n"); fiph = (struct fip_header *) skb->data; - FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "Received VLAN response... OP 0x%x SUB_OP 0x%x\n", ntohs(fiph->fip_op), fiph->fip_subcode); @@ -463,7 +469,7 @@ static void fnic_fcoe_process_vlan_resp(struct fnic *fnic, struct sk_buff *skb) if (list_empty(&fnic->vlans)) { /* retry from timer */ atomic64_inc(&fnic_stats->vlan_stats.resp_withno_vlanID); - FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "No VLAN descriptors in FIP VLAN response\n"); spin_unlock_irqrestore(&fnic->vlans_lock, flags); goto out; @@ -721,7 +727,8 @@ void fnic_update_mac_locked(struct fnic *fnic, u8 *new) new = ctl; if (ether_addr_equal(data, new)) return; - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, "update_mac %pM\n", new); + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, + "update_mac %pM\n", new); if (!is_zero_ether_addr(data) && !ether_addr_equal(data, ctl)) vnic_dev_del_addr(fnic->vdev, data); memcpy(data, new, ETH_ALEN); @@ -763,8 +770,9 @@ void fnic_set_port_id(struct fc_lport *lport, u32 port_id, struct fc_frame *fp) u8 *mac; int ret; - FNIC_FCS_DBG(KERN_DEBUG, lport->host, "set port_id %x fp %p\n", - port_id, fp); + FNIC_FCS_DBG(KERN_DEBUG, lport->host, fnic->fnic_num, + "set port_id 0x%x fp 0x%p\n", + port_id, fp); /* * If we're clearing the FC_ID, change to use the ctl_src_addr. @@ -790,10 +798,9 @@ void fnic_set_port_id(struct fc_lport *lport, u32 port_id, struct fc_frame *fp) if (fnic->state == FNIC_IN_ETH_MODE || fnic->state == FNIC_IN_FC_MODE) fnic->state = FNIC_IN_ETH_TRANS_FC_MODE; else { - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, - "Unexpected fnic state %s while" - " processing flogi resp\n", - fnic_state_to_str(fnic->state)); + FNIC_FCS_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "Unexpected fnic state: %s processing FLOGI response", + fnic_state_to_str(fnic->state)); spin_unlock_irq(&fnic->fnic_lock); return; } @@ -870,7 +877,7 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc skb_trim(skb, bytes_written); if (!fcs_ok) { atomic64_inc(&fnic_stats->misc_stats.frame_errors); - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fcs error. dropping packet.\n"); goto drop; } @@ -886,7 +893,7 @@ static void fnic_rq_cmpl_frame_recv(struct vnic_rq *rq, struct cq_desc if (!fcs_ok || packet_error || !fcoe_fc_crc_ok || fcoe_enc_error) { atomic64_inc(&fnic_stats->misc_stats.frame_errors); - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fnic rq_cmpl fcoe x%x fcsok x%x" " pkterr x%x fcoe_fc_crc_ok x%x, fcoe_enc_err" " x%x\n", @@ -967,7 +974,7 @@ int fnic_alloc_rq_frame(struct vnic_rq *rq) len = FC_FRAME_HEADROOM + FC_MAX_FRAME + FC_FRAME_TAILROOM; skb = dev_alloc_skb(len); if (!skb) { - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Unable to allocate RQ sk_buff\n"); return -ENOMEM; } @@ -1175,7 +1182,7 @@ int fnic_send(struct fc_lport *lp, struct fc_frame *fp) /** * fnic_flush_tx() - send queued frames. - * @fnic: fnic device + * @work: pointer to work element * * Send frames that were waiting to go out in FC or Ethernet mode. * Whenever changing modes we purge queued frames, so these frames should @@ -1183,8 +1190,9 @@ int fnic_send(struct fc_lport *lp, struct fc_frame *fp) * * Called without fnic_lock held. */ -void fnic_flush_tx(struct fnic *fnic) +void fnic_flush_tx(struct work_struct *work) { + struct fnic *fnic = container_of(work, struct fnic, flush_work); struct sk_buff *skb; struct fc_frame *fp; @@ -1341,12 +1349,12 @@ void fnic_handle_fip_timer(struct fnic *fnic) } vlan = list_first_entry(&fnic->vlans, struct fcoe_vlan, list); - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fip_timer: vlan %d state %d sol_count %d\n", vlan->vid, vlan->state, vlan->sol_count); switch (vlan->state) { case FIP_VLAN_USED: - FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_FCS_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "FIP VLAN is selected for FC transaction\n"); spin_unlock_irqrestore(&fnic->vlans_lock, flags); break; @@ -1365,7 +1373,7 @@ void fnic_handle_fip_timer(struct fnic *fnic) * no response on this vlan, remove from the list. * Try the next vlan */ - FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "Dequeue this VLAN ID %d from list\n", vlan->vid); list_del(&vlan->list); @@ -1375,7 +1383,7 @@ void fnic_handle_fip_timer(struct fnic *fnic) /* we exhausted all vlans, restart vlan disc */ spin_unlock_irqrestore(&fnic->vlans_lock, flags); - FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, + FNIC_FCS_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "fip_timer: vlan list empty, " "trigger vlan disc\n"); fnic_event_enq(fnic, FNIC_EVT_START_VLAN_DISC); diff --git a/drivers/scsi/fnic/fnic_isr.c b/drivers/scsi/fnic/fnic_isr.c index 8896758fed..ff85441c6c 100644 --- a/drivers/scsi/fnic/fnic_isr.c +++ b/drivers/scsi/fnic/fnic_isr.c @@ -38,8 +38,13 @@ static irqreturn_t fnic_isr_legacy(int irq, void *data) fnic_log_q_error(fnic); } + if (pba & (1 << FNIC_INTX_DUMMY)) { + atomic64_inc(&fnic->fnic_stats.misc_stats.intx_dummy); + vnic_intr_return_all_credits(&fnic->intr[FNIC_INTX_DUMMY]); + } + if (pba & (1 << FNIC_INTX_WQ_RQ_COPYWQ)) { - work_done += fnic_wq_copy_cmpl_handler(fnic, io_completions); + work_done += fnic_wq_copy_cmpl_handler(fnic, io_completions, FNIC_MQ_CQ_INDEX); work_done += fnic_wq_cmpl_handler(fnic, -1); work_done += fnic_rq_cmpl_handler(fnic, -1); @@ -60,7 +65,7 @@ static irqreturn_t fnic_isr_msi(int irq, void *data) fnic->fnic_stats.misc_stats.last_isr_time = jiffies; atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count); - work_done += fnic_wq_copy_cmpl_handler(fnic, io_completions); + work_done += fnic_wq_copy_cmpl_handler(fnic, io_completions, FNIC_MQ_CQ_INDEX); work_done += fnic_wq_cmpl_handler(fnic, -1); work_done += fnic_rq_cmpl_handler(fnic, -1); @@ -109,12 +114,22 @@ static irqreturn_t fnic_isr_msix_wq_copy(int irq, void *data) { struct fnic *fnic = data; unsigned long wq_copy_work_done = 0; + int i; fnic->fnic_stats.misc_stats.last_isr_time = jiffies; atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count); - wq_copy_work_done = fnic_wq_copy_cmpl_handler(fnic, io_completions); - vnic_intr_return_credits(&fnic->intr[FNIC_MSIX_WQ_COPY], + i = irq - fnic->msix[0].irq_num; + if (i >= fnic->wq_copy_count + fnic->copy_wq_base || + i < 0 || fnic->msix[i].irq_num != irq) { + for (i = fnic->copy_wq_base; i < fnic->wq_copy_count + fnic->copy_wq_base ; i++) { + if (fnic->msix[i].irq_num == irq) + break; + } + } + + wq_copy_work_done = fnic_wq_copy_cmpl_handler(fnic, io_completions, i); + vnic_intr_return_credits(&fnic->intr[i], wq_copy_work_done, 1 /* unmask intr */, 1 /* reset intr timer */); @@ -128,7 +143,7 @@ static irqreturn_t fnic_isr_msix_err_notify(int irq, void *data) fnic->fnic_stats.misc_stats.last_isr_time = jiffies; atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count); - vnic_intr_return_all_credits(&fnic->intr[FNIC_MSIX_ERR_NOTIFY]); + vnic_intr_return_all_credits(&fnic->intr[fnic->err_intr_offset]); fnic_log_q_error(fnic); fnic_handle_link_event(fnic); @@ -186,26 +201,30 @@ int fnic_request_intr(struct fnic *fnic) fnic->msix[FNIC_MSIX_WQ].isr = fnic_isr_msix_wq; fnic->msix[FNIC_MSIX_WQ].devid = fnic; - sprintf(fnic->msix[FNIC_MSIX_WQ_COPY].devname, - "%.11s-scsi-wq", fnic->name); - fnic->msix[FNIC_MSIX_WQ_COPY].isr = fnic_isr_msix_wq_copy; - fnic->msix[FNIC_MSIX_WQ_COPY].devid = fnic; + for (i = fnic->copy_wq_base; i < fnic->wq_copy_count + fnic->copy_wq_base; i++) { + sprintf(fnic->msix[i].devname, + "%.11s-scsi-wq-%d", fnic->name, i-FNIC_MSIX_WQ_COPY); + fnic->msix[i].isr = fnic_isr_msix_wq_copy; + fnic->msix[i].devid = fnic; + } - sprintf(fnic->msix[FNIC_MSIX_ERR_NOTIFY].devname, + sprintf(fnic->msix[fnic->err_intr_offset].devname, "%.11s-err-notify", fnic->name); - fnic->msix[FNIC_MSIX_ERR_NOTIFY].isr = + fnic->msix[fnic->err_intr_offset].isr = fnic_isr_msix_err_notify; - fnic->msix[FNIC_MSIX_ERR_NOTIFY].devid = fnic; + fnic->msix[fnic->err_intr_offset].devid = fnic; - for (i = 0; i < ARRAY_SIZE(fnic->msix); i++) { - err = request_irq(pci_irq_vector(fnic->pdev, i), - fnic->msix[i].isr, 0, - fnic->msix[i].devname, - fnic->msix[i].devid); + for (i = 0; i < fnic->intr_count; i++) { + fnic->msix[i].irq_num = pci_irq_vector(fnic->pdev, i); + + err = request_irq(fnic->msix[i].irq_num, + fnic->msix[i].isr, 0, + fnic->msix[i].devname, + fnic->msix[i].devid); if (err) { - shost_printk(KERN_ERR, fnic->lport->host, - "MSIX: request_irq" - " failed %d\n", err); + FNIC_ISR_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "request_irq failed with error: %d\n", + err); fnic_free_intr(fnic); break; } @@ -220,44 +239,99 @@ int fnic_request_intr(struct fnic *fnic) return err; } -int fnic_set_intr_mode(struct fnic *fnic) +int fnic_set_intr_mode_msix(struct fnic *fnic) { unsigned int n = ARRAY_SIZE(fnic->rq); unsigned int m = ARRAY_SIZE(fnic->wq); - unsigned int o = ARRAY_SIZE(fnic->wq_copy); + unsigned int o = ARRAY_SIZE(fnic->hw_copy_wq); + unsigned int min_irqs = n + m + 1 + 1; /*rq, raw wq, wq, err*/ /* - * Set interrupt mode (INTx, MSI, MSI-X) depending - * system capabilities. - * - * Try MSI-X first - * * We need n RQs, m WQs, o Copy WQs, n+m+o CQs, and n+m+o+1 INTRs * (last INTR is used for WQ/RQ errors and notification area) */ - if (fnic->rq_count >= n && - fnic->raw_wq_count >= m && - fnic->wq_copy_count >= o && - fnic->cq_count >= n + m + o) { - int vecs = n + m + o + 1; - - if (pci_alloc_irq_vectors(fnic->pdev, vecs, vecs, - PCI_IRQ_MSIX) == vecs) { + FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "rq-array size: %d wq-array size: %d copy-wq array size: %d\n", + n, m, o); + FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "rq_count: %d raw_wq_count: %d wq_copy_count: %d cq_count: %d\n", + fnic->rq_count, fnic->raw_wq_count, + fnic->wq_copy_count, fnic->cq_count); + + if (fnic->rq_count <= n && fnic->raw_wq_count <= m && + fnic->wq_copy_count <= o) { + int vec_count = 0; + int vecs = fnic->rq_count + fnic->raw_wq_count + fnic->wq_copy_count + 1; + + vec_count = pci_alloc_irq_vectors(fnic->pdev, min_irqs, vecs, + PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); + FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "allocated %d MSI-X vectors\n", + vec_count); + + if (vec_count > 0) { + if (vec_count < vecs) { + FNIC_ISR_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "interrupts number mismatch: vec_count: %d vecs: %d\n", + vec_count, vecs); + if (vec_count < min_irqs) { + FNIC_ISR_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "no interrupts for copy wq\n"); + return 1; + } + } + fnic->rq_count = n; fnic->raw_wq_count = m; - fnic->wq_copy_count = o; - fnic->wq_count = m + o; - fnic->cq_count = n + m + o; - fnic->intr_count = vecs; - fnic->err_intr_offset = FNIC_MSIX_ERR_NOTIFY; - - FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host, - "Using MSI-X Interrupts\n"); - vnic_dev_set_intr_mode(fnic->vdev, - VNIC_DEV_INTR_MODE_MSIX); + fnic->copy_wq_base = fnic->rq_count + fnic->raw_wq_count; + fnic->wq_copy_count = vec_count - n - m - 1; + fnic->wq_count = fnic->raw_wq_count + fnic->wq_copy_count; + if (fnic->cq_count != vec_count - 1) { + FNIC_ISR_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "CQ count: %d does not match MSI-X vector count: %d\n", + fnic->cq_count, vec_count); + fnic->cq_count = vec_count - 1; + } + fnic->intr_count = vec_count; + fnic->err_intr_offset = fnic->rq_count + fnic->wq_count; + + FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "rq_count: %d raw_wq_count: %d copy_wq_base: %d\n", + fnic->rq_count, + fnic->raw_wq_count, fnic->copy_wq_base); + + FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "wq_copy_count: %d wq_count: %d cq_count: %d\n", + fnic->wq_copy_count, + fnic->wq_count, fnic->cq_count); + + FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "intr_count: %d err_intr_offset: %u", + fnic->intr_count, + fnic->err_intr_offset); + + vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_MSIX); + FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "fnic using MSI-X\n"); return 0; } } + return 1; +} + +int fnic_set_intr_mode(struct fnic *fnic) +{ + int ret_status = 0; + + /* + * Set interrupt mode (INTx, MSI, MSI-X) depending + * system capabilities. + * + * Try MSI-X first + */ + ret_status = fnic_set_intr_mode_msix(fnic); + if (ret_status == 0) + return ret_status; /* * Next try MSI @@ -277,7 +351,7 @@ int fnic_set_intr_mode(struct fnic *fnic) fnic->intr_count = 1; fnic->err_intr_offset = 0; - FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Using MSI Interrupts\n"); vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_MSI); @@ -303,7 +377,7 @@ int fnic_set_intr_mode(struct fnic *fnic) fnic->cq_count = 3; fnic->intr_count = 3; - FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Using Legacy Interrupts\n"); vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_INTX); diff --git a/drivers/scsi/fnic/fnic_main.c b/drivers/scsi/fnic/fnic_main.c index f27f9319e0..29eead383e 100644 --- a/drivers/scsi/fnic/fnic_main.c +++ b/drivers/scsi/fnic/fnic_main.c @@ -12,9 +12,11 @@ #include <linux/pci.h> #include <linux/skbuff.h> #include <linux/interrupt.h> +#include <linux/irq.h> #include <linux/spinlock.h> #include <linux/workqueue.h> #include <linux/if_ether.h> +#include <linux/blk-mq-pci.h> #include <scsi/fc/fc_fip.h> #include <scsi/scsi_host.h> #include <scsi/scsi_transport.h> @@ -39,6 +41,7 @@ static struct kmem_cache *fnic_sgl_cache[FNIC_SGL_NUM_CACHES]; static struct kmem_cache *fnic_io_req_cache; static LIST_HEAD(fnic_list); static DEFINE_SPINLOCK(fnic_list_lock); +static DEFINE_IDA(fnic_ida); /* Supported devices by fnic module */ static struct pci_device_id fnic_id_table[] = { @@ -113,6 +116,7 @@ static const struct scsi_host_template fnic_host_template = { .shost_groups = fnic_host_groups, .track_queue_depth = 1, .cmd_size = sizeof(struct fnic_cmd_priv), + .map_queues = fnic_mq_map_queues_cpus, }; static void @@ -209,7 +213,7 @@ static struct fc_host_statistics *fnic_get_stats(struct Scsi_Host *host) spin_unlock_irqrestore(&fnic->fnic_lock, flags); if (ret) { - FNIC_MAIN_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_MAIN_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fnic: Get vnic stats failed" " 0x%x", ret); return stats; @@ -321,7 +325,7 @@ static void fnic_reset_host_stats(struct Scsi_Host *host) spin_unlock_irqrestore(&fnic->fnic_lock, flags); if (ret) { - FNIC_MAIN_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_MAIN_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fnic: Reset vnic stats failed" " 0x%x", ret); return; @@ -354,7 +358,7 @@ void fnic_log_q_error(struct fnic *fnic) } for (i = 0; i < fnic->wq_copy_count; i++) { - error_status = ioread32(&fnic->wq_copy[i].ctrl->error_status); + error_status = ioread32(&fnic->hw_copy_wq[i].ctrl->error_status); if (error_status) shost_printk(KERN_ERR, fnic->lport->host, "CWQ[%d] error_status" @@ -389,7 +393,7 @@ static int fnic_notify_set(struct fnic *fnic) err = vnic_dev_notify_set(fnic->vdev, -1); break; case VNIC_DEV_INTR_MODE_MSIX: - err = vnic_dev_notify_set(fnic->vdev, FNIC_MSIX_ERR_NOTIFY); + err = vnic_dev_notify_set(fnic->vdev, fnic->wq_copy_count + fnic->copy_wq_base); break; default: shost_printk(KERN_ERR, fnic->lport->host, @@ -475,6 +479,7 @@ static int fnic_cleanup(struct fnic *fnic) { unsigned int i; int err; + int raw_wq_rq_counts; vnic_dev_disable(fnic->vdev); for (i = 0; i < fnic->intr_count; i++) @@ -491,13 +496,14 @@ static int fnic_cleanup(struct fnic *fnic) return err; } for (i = 0; i < fnic->wq_copy_count; i++) { - err = vnic_wq_copy_disable(&fnic->wq_copy[i]); + err = vnic_wq_copy_disable(&fnic->hw_copy_wq[i]); if (err) return err; + raw_wq_rq_counts = fnic->raw_wq_count + fnic->rq_count; + fnic_wq_copy_cmpl_handler(fnic, -1, i + raw_wq_rq_counts); } /* Clean up completed IOs and FCS frames */ - fnic_wq_copy_cmpl_handler(fnic, io_completions); fnic_wq_cmpl_handler(fnic, -1); fnic_rq_cmpl_handler(fnic, -1); @@ -507,7 +513,7 @@ static int fnic_cleanup(struct fnic *fnic) for (i = 0; i < fnic->rq_count; i++) vnic_rq_clean(&fnic->rq[i], fnic_free_rq_buf); for (i = 0; i < fnic->wq_copy_count; i++) - vnic_wq_copy_clean(&fnic->wq_copy[i], + vnic_wq_copy_clean(&fnic->hw_copy_wq[i], fnic_wq_copy_cleanup_handler); for (i = 0; i < fnic->cq_count; i++) @@ -560,11 +566,6 @@ static int fnic_scsi_drv_init(struct fnic *fnic) host->max_cmd_len = FCOE_MAX_CMD_LEN; host->nr_hw_queues = fnic->wq_copy_count; - if (host->nr_hw_queues > 1) - shost_printk(KERN_ERR, host, - "fnic: blk-mq is not supported"); - - host->nr_hw_queues = fnic->wq_copy_count = 1; shost_printk(KERN_INFO, host, "fnic: can_queue: %d max_lun: %llu", @@ -577,15 +578,43 @@ static int fnic_scsi_drv_init(struct fnic *fnic) return 0; } +void fnic_mq_map_queues_cpus(struct Scsi_Host *host) +{ + struct fc_lport *lp = shost_priv(host); + struct fnic *fnic = lport_priv(lp); + struct pci_dev *l_pdev = fnic->pdev; + int intr_mode = fnic->config.intr_mode; + struct blk_mq_queue_map *qmap = &host->tag_set.map[HCTX_TYPE_DEFAULT]; + + if (intr_mode == VNIC_DEV_INTR_MODE_MSI || intr_mode == VNIC_DEV_INTR_MODE_INTX) { + FNIC_MAIN_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "intr_mode is not msix\n"); + return; + } + + FNIC_MAIN_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "qmap->nr_queues: %d\n", qmap->nr_queues); + + if (l_pdev == NULL) { + FNIC_MAIN_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "l_pdev is null\n"); + return; + } + + blk_mq_pci_map_queues(qmap, l_pdev, FNIC_PCI_OFFSET); +} + static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct Scsi_Host *host; struct fc_lport *lp; struct fnic *fnic; mempool_t *pool; - int err; + int err = 0; + int fnic_id = 0; int i; unsigned long flags; + int hwq; /* * Allocate SCSI Host and set up association between host, @@ -597,25 +626,28 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err = -ENOMEM; goto err_out; } + host = lp->host; fnic = lport_priv(lp); + + fnic_id = ida_alloc(&fnic_ida, GFP_KERNEL); + if (fnic_id < 0) { + pr_err("Unable to alloc fnic ID\n"); + err = fnic_id; + goto err_out_ida_alloc; + } fnic->lport = lp; fnic->ctlr.lp = lp; - fnic->link_events = 0; + fnic->pdev = pdev; snprintf(fnic->name, sizeof(fnic->name) - 1, "%s%d", DRV_NAME, host->host_no); host->transportt = fnic_fc_transport; - + fnic->fnic_num = fnic_id; fnic_stats_debugfs_init(fnic); - /* Setup PCI resources */ - pci_set_drvdata(pdev, fnic); - - fnic->pdev = pdev; - err = pci_enable_device(pdev); if (err) { shost_printk(KERN_ERR, fnic->lport->host, @@ -717,7 +749,8 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_out_dev_close; } - fnic_scsi_drv_init(fnic); + /* Setup PCI resources */ + pci_set_drvdata(pdev, fnic); fnic_get_res_counts(fnic); @@ -737,6 +770,16 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_out_clear_intr; } + fnic_scsi_drv_init(fnic); + + for (hwq = 0; hwq < fnic->wq_copy_count; hwq++) { + fnic->sw_copy_wq[hwq].ioreq_table_size = fnic->fnic_max_tag_id; + fnic->sw_copy_wq[hwq].io_req_table = + kzalloc((fnic->sw_copy_wq[hwq].ioreq_table_size + 1) * + sizeof(struct fnic_io_req *), GFP_KERNEL); + } + shost_printk(KERN_INFO, fnic->lport->host, "fnic copy wqs: %d, Q0 ioreq table size: %d\n", + fnic->wq_copy_count, fnic->sw_copy_wq[0].ioreq_table_size); /* initialize all fnic locks */ spin_lock_init(&fnic->fnic_lock); @@ -751,11 +794,6 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) fnic->fw_ack_index[i] = -1; } - for (i = 0; i < FNIC_IO_LOCKS; i++) - spin_lock_init(&fnic->io_req_lock[i]); - - spin_lock_init(&fnic->sgreset_lock); - err = -ENOMEM; fnic->io_req_pool = mempool_create_slab_pool(2, fnic_io_req_cache); if (!fnic->io_req_pool) @@ -792,6 +830,7 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) spin_lock_init(&fnic->vlans_lock); INIT_WORK(&fnic->fip_frame_work, fnic_handle_fip_frame); INIT_WORK(&fnic->event_work, fnic_handle_event); + INIT_WORK(&fnic->flush_work, fnic_flush_tx); skb_queue_head_init(&fnic->fip_frame_queue); INIT_LIST_HEAD(&fnic->evlist); INIT_LIST_HEAD(&fnic->vlans); @@ -823,16 +862,32 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* allocate RQ buffers and post them to RQ*/ for (i = 0; i < fnic->rq_count; i++) { - vnic_rq_enable(&fnic->rq[i]); err = vnic_rq_fill(&fnic->rq[i], fnic_alloc_rq_frame); if (err) { shost_printk(KERN_ERR, fnic->lport->host, "fnic_alloc_rq_frame can't alloc " "frame\n"); - goto err_out_free_rq_buf; + goto err_out_rq_buf; } } + /* Enable all queues */ + for (i = 0; i < fnic->raw_wq_count; i++) + vnic_wq_enable(&fnic->wq[i]); + for (i = 0; i < fnic->rq_count; i++) { + if (!ioread32(&fnic->rq[i].ctrl->enable)) + vnic_rq_enable(&fnic->rq[i]); + } + for (i = 0; i < fnic->wq_copy_count; i++) + vnic_wq_copy_enable(&fnic->hw_copy_wq[i]); + + err = fnic_request_intr(fnic); + if (err) { + shost_printk(KERN_ERR, fnic->lport->host, + "Unable to request irq.\n"); + goto err_out_request_intr; + } + /* * Initialization done with PCI system, hardware, firmware. * Add host to SCSI @@ -841,9 +896,10 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) { shost_printk(KERN_ERR, fnic->lport->host, "fnic: scsi_add_host failed...exiting\n"); - goto err_out_free_rq_buf; + goto err_out_scsi_add_host; } + /* Start local port initiatialization */ lp->link_up = 0; @@ -867,7 +923,7 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (!fc_exch_mgr_alloc(lp, FC_CLASS_3, FCPIO_HOST_EXCH_RANGE_START, FCPIO_HOST_EXCH_RANGE_END, NULL)) { err = -ENOMEM; - goto err_out_remove_scsi_host; + goto err_out_fc_exch_mgr_alloc; } fc_lport_init_stats(lp); @@ -895,21 +951,8 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) skb_queue_head_init(&fnic->frame_queue); skb_queue_head_init(&fnic->tx_queue); - /* Enable all queues */ - for (i = 0; i < fnic->raw_wq_count; i++) - vnic_wq_enable(&fnic->wq[i]); - for (i = 0; i < fnic->wq_copy_count; i++) - vnic_wq_copy_enable(&fnic->wq_copy[i]); - fc_fabric_login(lp); - err = fnic_request_intr(fnic); - if (err) { - shost_printk(KERN_ERR, fnic->lport->host, - "Unable to request irq.\n"); - goto err_out_free_exch_mgr; - } - vnic_dev_enable(fnic->vdev); for (i = 0; i < fnic->intr_count; i++) @@ -921,12 +964,15 @@ static int fnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err_out_free_exch_mgr: fc_exch_mgr_free(lp); -err_out_remove_scsi_host: +err_out_fc_exch_mgr_alloc: fc_remove_host(lp->host); scsi_remove_host(lp->host); -err_out_free_rq_buf: +err_out_scsi_add_host: + fnic_free_intr(fnic); +err_out_request_intr: for (i = 0; i < fnic->rq_count; i++) vnic_rq_clean(&fnic->rq[i], fnic_free_rq_buf); +err_out_rq_buf: vnic_dev_notify_unset(fnic->vdev); err_out_free_max_pool: mempool_destroy(fnic->io_sgl_pool[FNIC_SGL_CACHE_MAX]); @@ -935,6 +981,8 @@ err_out_free_dflt_pool: err_out_free_ioreq_pool: mempool_destroy(fnic->io_req_pool); err_out_free_resources: + for (hwq = 0; hwq < fnic->wq_copy_count; hwq++) + kfree(fnic->sw_copy_wq[hwq].io_req_table); fnic_free_vnic_resources(fnic); err_out_clear_intr: fnic_clear_intr_mode(fnic); @@ -951,6 +999,8 @@ err_out_disable_device: pci_disable_device(pdev); err_out_free_hba: fnic_stats_debugfs_remove(fnic); + ida_free(&fnic_ida, fnic->fnic_num); +err_out_ida_alloc: scsi_host_put(lp->host); err_out: return err; @@ -961,6 +1011,7 @@ static void fnic_remove(struct pci_dev *pdev) struct fnic *fnic = pci_get_drvdata(pdev); struct fc_lport *lp = fnic->lport; unsigned long flags; + int hwq; /* * Mark state so that the workqueue thread stops forwarding @@ -1021,6 +1072,8 @@ static void fnic_remove(struct pci_dev *pdev) fc_remove_host(fnic->lport->host); scsi_remove_host(fnic->lport->host); + for (hwq = 0; hwq < fnic->wq_copy_count; hwq++) + kfree(fnic->sw_copy_wq[hwq].io_req_table); fc_exch_mgr_free(fnic->lport); vnic_dev_notify_unset(fnic->vdev); fnic_free_intr(fnic); @@ -1031,6 +1084,7 @@ static void fnic_remove(struct pci_dev *pdev) fnic_iounmap(fnic); pci_release_regions(pdev); pci_disable_device(pdev); + ida_free(&fnic_ida, fnic->fnic_num); scsi_host_put(lp->host); } @@ -1168,6 +1222,7 @@ static void __exit fnic_cleanup_module(void) fnic_trace_free(); fnic_fc_trace_free(); fnic_debugfs_terminate(); + ida_destroy(&fnic_ida); } module_init(fnic_init_module); diff --git a/drivers/scsi/fnic/fnic_res.c b/drivers/scsi/fnic/fnic_res.c index a1c9cfcace..33dd27f6f2 100644 --- a/drivers/scsi/fnic/fnic_res.c +++ b/drivers/scsi/fnic/fnic_res.c @@ -57,6 +57,8 @@ int fnic_get_vnic_config(struct fnic *fnic) GET_CONFIG(port_down_timeout); GET_CONFIG(port_down_io_retries); GET_CONFIG(luns_per_tgt); + GET_CONFIG(intr_mode); + GET_CONFIG(wq_copy_count); c->wq_enet_desc_count = min_t(u32, VNIC_FNIC_WQ_DESCS_MAX, @@ -131,6 +133,12 @@ int fnic_get_vnic_config(struct fnic *fnic) c->intr_timer = min_t(u16, VNIC_INTR_TIMER_MAX, c->intr_timer); c->intr_timer_type = c->intr_timer_type; + /* for older firmware, GET_CONFIG will not return anything */ + if (c->wq_copy_count == 0) + c->wq_copy_count = 1; + + c->wq_copy_count = min_t(u16, FNIC_WQ_COPY_MAX, c->wq_copy_count); + shost_printk(KERN_INFO, fnic->lport->host, "vNIC MAC addr %pM " "wq/wq_copy/rq %d/%d/%d\n", @@ -161,6 +169,10 @@ int fnic_get_vnic_config(struct fnic *fnic) shost_printk(KERN_INFO, fnic->lport->host, "vNIC port dn io retries %d port dn timeout %d\n", c->port_down_io_retries, c->port_down_timeout); + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC wq_copy_count: %d\n", c->wq_copy_count); + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC intr mode: %d\n", c->intr_mode); return 0; } @@ -187,12 +199,25 @@ int fnic_set_nic_config(struct fnic *fnic, u8 rss_default_cpu, void fnic_get_res_counts(struct fnic *fnic) { fnic->wq_count = vnic_dev_get_res_count(fnic->vdev, RES_TYPE_WQ); - fnic->raw_wq_count = fnic->wq_count - 1; - fnic->wq_copy_count = fnic->wq_count - fnic->raw_wq_count; + fnic->raw_wq_count = 1; + fnic->wq_copy_count = fnic->config.wq_copy_count; fnic->rq_count = vnic_dev_get_res_count(fnic->vdev, RES_TYPE_RQ); fnic->cq_count = vnic_dev_get_res_count(fnic->vdev, RES_TYPE_CQ); fnic->intr_count = vnic_dev_get_res_count(fnic->vdev, RES_TYPE_INTR_CTRL); + + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC fw resources wq_count: %d\n", fnic->wq_count); + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC fw resources raw_wq_count: %d\n", fnic->raw_wq_count); + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC fw resources wq_copy_count: %d\n", fnic->wq_copy_count); + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC fw resources rq_count: %d\n", fnic->rq_count); + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC fw resources cq_count: %d\n", fnic->cq_count); + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC fw resources intr_count: %d\n", fnic->intr_count); } void fnic_free_vnic_resources(struct fnic *fnic) @@ -203,7 +228,7 @@ void fnic_free_vnic_resources(struct fnic *fnic) vnic_wq_free(&fnic->wq[i]); for (i = 0; i < fnic->wq_copy_count; i++) - vnic_wq_copy_free(&fnic->wq_copy[i]); + vnic_wq_copy_free(&fnic->hw_copy_wq[i]); for (i = 0; i < fnic->rq_count; i++) vnic_rq_free(&fnic->rq[i]); @@ -234,10 +259,15 @@ int fnic_alloc_vnic_resources(struct fnic *fnic) intr_mode == VNIC_DEV_INTR_MODE_MSIX ? "MSI-X" : "unknown"); - shost_printk(KERN_INFO, fnic->lport->host, "vNIC resources avail: " - "wq %d cp_wq %d raw_wq %d rq %d cq %d intr %d\n", - fnic->wq_count, fnic->wq_copy_count, fnic->raw_wq_count, - fnic->rq_count, fnic->cq_count, fnic->intr_count); + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC resources avail: wq %d cp_wq %d raw_wq %d rq %d", + fnic->wq_count, fnic->wq_copy_count, + fnic->raw_wq_count, fnic->rq_count); + + shost_printk(KERN_INFO, fnic->lport->host, + "vNIC resources avail: cq %d intr %d cpy-wq desc count %d\n", + fnic->cq_count, fnic->intr_count, + fnic->config.wq_copy_desc_count); /* Allocate Raw WQ used for FCS frames */ for (i = 0; i < fnic->raw_wq_count; i++) { @@ -250,7 +280,7 @@ int fnic_alloc_vnic_resources(struct fnic *fnic) /* Allocate Copy WQs used for SCSI IOs */ for (i = 0; i < fnic->wq_copy_count; i++) { - err = vnic_wq_copy_alloc(fnic->vdev, &fnic->wq_copy[i], + err = vnic_wq_copy_alloc(fnic->vdev, &fnic->hw_copy_wq[i], (fnic->raw_wq_count + i), fnic->config.wq_copy_desc_count, sizeof(struct fcpio_host_req)); @@ -357,7 +387,7 @@ int fnic_alloc_vnic_resources(struct fnic *fnic) } for (i = 0; i < fnic->wq_copy_count; i++) { - vnic_wq_copy_init(&fnic->wq_copy[i], + vnic_wq_copy_init(&fnic->hw_copy_wq[i], 0 /* cq_index 0 - always */, error_interrupt_enable, error_interrupt_offset); diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c index 416d819548..fc4cee91b1 100644 --- a/drivers/scsi/fnic/fnic_scsi.c +++ b/drivers/scsi/fnic/fnic_scsi.c @@ -92,20 +92,6 @@ static const char *fnic_fcpio_status_to_str(unsigned int status) static void fnic_cleanup_io(struct fnic *fnic); -static inline spinlock_t *fnic_io_lock_hash(struct fnic *fnic, - struct scsi_cmnd *sc) -{ - u32 hash = scsi_cmd_to_rq(sc)->tag & (FNIC_IO_LOCKS - 1); - - return &fnic->io_req_lock[hash]; -} - -static inline spinlock_t *fnic_io_lock_tag(struct fnic *fnic, - int tag) -{ - return &fnic->io_req_lock[tag & (FNIC_IO_LOCKS - 1)]; -} - /* * Unmap the data buffer and sense buffer for an io_req, * also unmap and free the device-private scatter/gather list. @@ -129,23 +115,23 @@ static void fnic_release_ioreq_buf(struct fnic *fnic, } /* Free up Copy Wq descriptors. Called with copy_wq lock held */ -static int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq) +static int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq, unsigned int hwq) { /* if no Ack received from firmware, then nothing to clean */ - if (!fnic->fw_ack_recd[0]) + if (!fnic->fw_ack_recd[hwq]) return 1; /* * Update desc_available count based on number of freed descriptors * Account for wraparound */ - if (wq->to_clean_index <= fnic->fw_ack_index[0]) - wq->ring.desc_avail += (fnic->fw_ack_index[0] + if (wq->to_clean_index <= fnic->fw_ack_index[hwq]) + wq->ring.desc_avail += (fnic->fw_ack_index[hwq] - wq->to_clean_index + 1); else wq->ring.desc_avail += (wq->ring.desc_count - wq->to_clean_index - + fnic->fw_ack_index[0] + 1); + + fnic->fw_ack_index[hwq] + 1); /* * just bump clean index to ack_index+1 accounting for wraparound @@ -153,10 +139,10 @@ static int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq) * to_clean_index and fw_ack_index, both inclusive */ wq->to_clean_index = - (fnic->fw_ack_index[0] + 1) % wq->ring.desc_count; + (fnic->fw_ack_index[hwq] + 1) % wq->ring.desc_count; /* we have processed the acks received so far */ - fnic->fw_ack_recd[0] = 0; + fnic->fw_ack_recd[hwq] = 0; return 0; } @@ -170,17 +156,14 @@ __fnic_set_state_flags(struct fnic *fnic, unsigned long st_flags, unsigned long clearbits) { unsigned long flags = 0; - unsigned long host_lock_flags = 0; spin_lock_irqsave(&fnic->fnic_lock, flags); - spin_lock_irqsave(fnic->lport->host->host_lock, host_lock_flags); if (clearbits) fnic->state_flags &= ~st_flags; else fnic->state_flags |= st_flags; - spin_unlock_irqrestore(fnic->lport->host->host_lock, host_lock_flags); spin_unlock_irqrestore(&fnic->fnic_lock, flags); return; @@ -193,7 +176,7 @@ __fnic_set_state_flags(struct fnic *fnic, unsigned long st_flags, */ int fnic_fw_reset_handler(struct fnic *fnic) { - struct vnic_wq_copy *wq = &fnic->wq_copy[0]; + struct vnic_wq_copy *wq = &fnic->hw_copy_wq[0]; int ret = 0; unsigned long flags; @@ -210,7 +193,7 @@ int fnic_fw_reset_handler(struct fnic *fnic) spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + free_wq_copy_descs(fnic, wq, 0); if (!vnic_wq_copy_desc_avail(wq)) ret = -EAGAIN; @@ -228,12 +211,12 @@ int fnic_fw_reset_handler(struct fnic *fnic) if (!ret) { atomic64_inc(&fnic->fnic_stats.reset_stats.fw_resets); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Issued fw reset\n"); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "Issued fw reset\n"); } else { fnic_clear_state_flags(fnic, FNIC_FLAGS_FWRESET); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Failed to issue fw reset\n"); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "Failed to issue fw reset\n"); } return ret; @@ -246,7 +229,7 @@ int fnic_fw_reset_handler(struct fnic *fnic) */ int fnic_flogi_reg_handler(struct fnic *fnic, u32 fc_id) { - struct vnic_wq_copy *wq = &fnic->wq_copy[0]; + struct vnic_wq_copy *wq = &fnic->hw_copy_wq[0]; enum fcpio_flogi_reg_format_type format; struct fc_lport *lp = fnic->lport; u8 gw_mac[ETH_ALEN]; @@ -256,7 +239,7 @@ int fnic_flogi_reg_handler(struct fnic *fnic, u32 fc_id) spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + free_wq_copy_descs(fnic, wq, 0); if (!vnic_wq_copy_desc_avail(wq)) { ret = -EAGAIN; @@ -276,15 +259,15 @@ int fnic_flogi_reg_handler(struct fnic *fnic, u32 fc_id) fc_id, gw_mac, fnic->data_src_addr, lp->r_a_tov, lp->e_d_tov); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "FLOGI FIP reg issued fcid %x src %pM dest %pM\n", fc_id, fnic->data_src_addr, gw_mac); } else { fnic_queue_wq_copy_desc_flogi_reg(wq, SCSI_NO_TAG, format, fc_id, gw_mac); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "FLOGI reg issued fcid %x map %d dest %pM\n", - fc_id, fnic->ctlr.map_dest, gw_mac); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "FLOGI reg issued fcid 0x%x map %d dest 0x%p\n", + fc_id, fnic->ctlr.map_dest, gw_mac); } atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs); @@ -306,7 +289,9 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, struct vnic_wq_copy *wq, struct fnic_io_req *io_req, struct scsi_cmnd *sc, - int sg_count) + int sg_count, + uint32_t mqtag, + uint16_t hwq) { struct scatterlist *sg; struct fc_rport *rport = starget_to_rport(scsi_target(sc->device)); @@ -314,7 +299,6 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, struct host_sg_desc *desc; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; unsigned int i; - unsigned long intr_flags; int flags; u8 exch_flags; struct scsi_lun fc_lun; @@ -354,14 +338,11 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, int_to_scsilun(sc->device->lun, &fc_lun); /* Enqueue the descriptor in the Copy WQ */ - spin_lock_irqsave(&fnic->wq_copy_lock[0], intr_flags); - - if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[hwq]) + free_wq_copy_descs(fnic, wq, hwq); if (unlikely(!vnic_wq_copy_desc_avail(wq))) { - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags); - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "fnic_queue_wq_copy_desc failure - no descriptors\n"); atomic64_inc(&misc_stats->io_cpwq_alloc_failures); return SCSI_MLQUEUE_HOST_BUSY; @@ -378,7 +359,7 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, (rp->flags & FC_RP_FLAGS_RETRY)) exch_flags |= FCPIO_ICMND_SRFLAG_RETRY; - fnic_queue_wq_copy_desc_icmnd_16(wq, scsi_cmd_to_rq(sc)->tag, + fnic_queue_wq_copy_desc_icmnd_16(wq, mqtag, 0, exch_flags, io_req->sgl_cnt, SCSI_SENSE_BUFFERSIZE, io_req->sgl_list_pa, @@ -399,43 +380,52 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs, atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs)); - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags); return 0; } -/* - * fnic_queuecommand - * Routine to send a scsi cdb - * Called with host_lock held and interrupts disabled. - */ -static int fnic_queuecommand_lck(struct scsi_cmnd *sc) +int fnic_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *sc) { + struct request *const rq = scsi_cmd_to_rq(sc); + uint32_t mqtag = 0; void (*done)(struct scsi_cmnd *) = scsi_done; - const int tag = scsi_cmd_to_rq(sc)->tag; struct fc_lport *lp = shost_priv(sc->device->host); struct fc_rport *rport; struct fnic_io_req *io_req = NULL; struct fnic *fnic = lport_priv(lp); struct fnic_stats *fnic_stats = &fnic->fnic_stats; struct vnic_wq_copy *wq; - int ret; + int ret = 1; u64 cmd_trace; int sg_count = 0; unsigned long flags = 0; unsigned long ptr; - spinlock_t *io_lock = NULL; int io_lock_acquired = 0; struct fc_rport_libfc_priv *rp; + uint16_t hwq = 0; + + mqtag = blk_mq_unique_tag(rq); + spin_lock_irqsave(&fnic->fnic_lock, flags); - if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) + if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "fnic IO blocked flags: 0x%lx. Returning SCSI_MLQUEUE_HOST_BUSY\n", + fnic->state_flags); return SCSI_MLQUEUE_HOST_BUSY; + } - if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_FWRESET))) + if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_FWRESET))) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "fnic flags: 0x%lx. Returning SCSI_MLQUEUE_HOST_BUSY\n", + fnic->state_flags); return SCSI_MLQUEUE_HOST_BUSY; + } rport = starget_to_rport(scsi_target(sc->device)); if (!rport) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "returning DID_NO_CONNECT for IO as rport is NULL\n"); sc->result = DID_NO_CONNECT << 16; done(sc); @@ -444,7 +434,8 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc) ret = fc_remote_port_chkready(rport); if (ret) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "rport is not ready\n"); atomic64_inc(&fnic_stats->misc_stats.rport_not_ready); sc->result = ret; @@ -454,7 +445,8 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc) rp = rport->dd_data; if (!rp || rp->rp_state == RPORT_ST_DELETE) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "rport 0x%x removed, returning DID_NO_CONNECT\n", rport->port_id); @@ -465,7 +457,8 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc) } if (rp->rp_state != RPORT_ST_READY) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "rport 0x%x in state 0x%x, returning DID_IMM_RETRY\n", rport->port_id, rp->rp_state); @@ -474,17 +467,17 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc) return 0; } - if (lp->state != LPORT_ST_READY || !(lp->link_up)) + if (lp->state != LPORT_ST_READY || !(lp->link_up)) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "state not ready: %d/link not up: %d Returning HOST_BUSY\n", + lp->state, lp->link_up); return SCSI_MLQUEUE_HOST_BUSY; + } atomic_inc(&fnic->in_flight); - /* - * Release host lock, use driver resource specific locks from here. - * Don't re-enable interrupts in case they were disabled prior to the - * caller disabling them. - */ - spin_unlock(lp->host->host_lock); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); fnic_priv(sc)->state = FNIC_IOREQ_NOT_INITED; fnic_priv(sc)->flags = FNIC_NO_FLAGS; @@ -501,7 +494,7 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc) sg_count = scsi_dma_map(sc); if (sg_count < 0) { FNIC_TRACE(fnic_queuecommand, sc->device->host->host_no, - tag, sc, 0, sc->cmnd[0], sg_count, fnic_priv(sc)->state); + mqtag, sc, 0, sc->cmnd[0], sg_count, fnic_priv(sc)->state); mempool_free(io_req, fnic->io_req_pool); goto out; } @@ -536,11 +529,10 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc) } /* - * Will acquire lock defore setting to IO initialized. + * Will acquire lock before setting to IO initialized. */ - - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); + hwq = blk_mq_unique_tag_to_hwq(mqtag); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); /* initialize rest of io_req */ io_lock_acquired = 1; @@ -549,28 +541,40 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc) fnic_priv(sc)->state = FNIC_IOREQ_CMD_PENDING; fnic_priv(sc)->io_req = io_req; fnic_priv(sc)->flags |= FNIC_IO_INITIALIZED; + io_req->sc = sc; + + if (fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] != NULL) { + WARN(1, "fnic<%d>: %s: hwq: %d tag 0x%x already exists\n", + fnic->fnic_num, __func__, hwq, blk_mq_unique_tag_to_tag(mqtag)); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return SCSI_MLQUEUE_HOST_BUSY; + } + + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] = io_req; + io_req->tag = mqtag; /* create copy wq desc and enqueue it */ - wq = &fnic->wq_copy[0]; - ret = fnic_queue_wq_copy_desc(fnic, wq, io_req, sc, sg_count); + wq = &fnic->hw_copy_wq[hwq]; + atomic64_inc(&fnic_stats->io_stats.ios[hwq]); + ret = fnic_queue_wq_copy_desc(fnic, wq, io_req, sc, sg_count, mqtag, hwq); if (ret) { /* * In case another thread cancelled the request, * refetch the pointer under the lock. */ FNIC_TRACE(fnic_queuecommand, sc->device->host->host_no, - tag, sc, 0, 0, 0, fnic_flags_and_state(sc)); + mqtag, sc, 0, 0, 0, fnic_flags_and_state(sc)); io_req = fnic_priv(sc)->io_req; fnic_priv(sc)->io_req = NULL; + if (io_req) + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] = NULL; fnic_priv(sc)->state = FNIC_IOREQ_CMD_COMPLETE; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (io_req) { fnic_release_ioreq_buf(fnic, io_req, sc); mempool_free(io_req, fnic->io_req_pool); } atomic_dec(&fnic->in_flight); - /* acquire host lock before returning to SCSI */ - spin_lock(lp->host->host_lock); return ret; } else { atomic64_inc(&fnic_stats->io_stats.active_ios); @@ -590,20 +594,17 @@ out: sc->cmnd[5]); FNIC_TRACE(fnic_queuecommand, sc->device->host->host_no, - tag, sc, io_req, sg_count, cmd_trace, + mqtag, sc, io_req, sg_count, cmd_trace, fnic_flags_and_state(sc)); /* if only we issued IO, will we have the io lock */ if (io_lock_acquired) - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); - /* acquire host lock before returning to SCSI */ - spin_lock(lp->host->host_lock); return ret; } -DEF_SCSI_QCMD(fnic_queuecommand) /* * fnic_fcpio_fw_reset_cmpl_handler @@ -636,15 +637,14 @@ static int fnic_fcpio_fw_reset_cmpl_handler(struct fnic *fnic, if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE) { /* Check status of reset completion */ if (!hdr_status) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "reset cmpl success\n"); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "reset cmpl success\n"); /* Ready to send flogi out */ fnic->state = FNIC_IN_ETH_MODE; } else { - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "fnic fw_reset : failed %s\n", - fnic_fcpio_status_to_str(hdr_status)); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "reset failed with header status: %s\n", + fnic_fcpio_status_to_str(hdr_status)); /* * Unable to change to eth mode, cannot send out flogi @@ -657,10 +657,9 @@ static int fnic_fcpio_fw_reset_cmpl_handler(struct fnic *fnic, ret = -1; } } else { - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "Unexpected state %s while processing" - " reset cmpl\n", fnic_state_to_str(fnic->state)); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "Unexpected state while processing reset completion: %s\n", + fnic_state_to_str(fnic->state)); atomic64_inc(&reset_stats->fw_reset_failures); ret = -1; } @@ -681,7 +680,7 @@ static int fnic_fcpio_fw_reset_cmpl_handler(struct fnic *fnic, spin_unlock_irqrestore(&fnic->fnic_lock, flags); - fnic_flush_tx(fnic); + queue_work(fnic_event_queue, &fnic->flush_work); reset_cmpl_handler_end: fnic_clear_state_flags(fnic, FNIC_FLAGS_FWRESET); @@ -711,19 +710,19 @@ static int fnic_fcpio_flogi_reg_cmpl_handler(struct fnic *fnic, /* Check flogi registration completion status */ if (!hdr_status) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "flog reg succeeded\n"); fnic->state = FNIC_IN_FC_MODE; } else { FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, + fnic->lport->host, fnic->fnic_num, "fnic flogi reg :failed %s\n", fnic_fcpio_status_to_str(hdr_status)); fnic->state = FNIC_IN_ETH_MODE; ret = -1; } } else { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Unexpected fnic state %s while" " processing flogi reg completion\n", fnic_state_to_str(fnic->state)); @@ -737,7 +736,7 @@ static int fnic_fcpio_flogi_reg_cmpl_handler(struct fnic *fnic, } spin_unlock_irqrestore(&fnic->fnic_lock, flags); - fnic_flush_tx(fnic); + queue_work(fnic_event_queue, &fnic->flush_work); queue_work(fnic_event_queue, &fnic->frame_work); } else { spin_unlock_irqrestore(&fnic->fnic_lock, flags); @@ -780,20 +779,21 @@ static inline void fnic_fcpio_ack_handler(struct fnic *fnic, u16 request_out = desc->u.ack.request_out; unsigned long flags; u64 *ox_id_tag = (u64 *)(void *)desc; + unsigned int wq_index = cq_index; /* mark the ack state */ - wq = &fnic->wq_copy[cq_index - fnic->raw_wq_count - fnic->rq_count]; - spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); + wq = &fnic->hw_copy_wq[cq_index]; + spin_lock_irqsave(&fnic->wq_copy_lock[wq_index], flags); fnic->fnic_stats.misc_stats.last_ack_time = jiffies; if (is_ack_index_in_range(wq, request_out)) { - fnic->fw_ack_index[0] = request_out; - fnic->fw_ack_recd[0] = 1; + fnic->fw_ack_index[wq_index] = request_out; + fnic->fw_ack_recd[wq_index] = 1; } else atomic64_inc( &fnic->fnic_stats.misc_stats.ack_index_out_of_range); - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[wq_index], flags); FNIC_TRACE(fnic_fcpio_ack_handler, fnic->lport->host->host_no, 0, 0, ox_id_tag[2], ox_id_tag[3], ox_id_tag[4], ox_id_tag[5]); @@ -803,12 +803,12 @@ static inline void fnic_fcpio_ack_handler(struct fnic *fnic, * fnic_fcpio_icmnd_cmpl_handler * Routine to handle icmnd completions */ -static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, +static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, unsigned int cq_index, struct fcpio_fw_req *desc) { u8 type; u8 hdr_status; - struct fcpio_tag tag; + struct fcpio_tag ftag; u32 id; u64 xfer_len = 0; struct fcpio_icmnd_cmpl *icmnd_cmpl; @@ -816,27 +816,47 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, struct scsi_cmnd *sc; struct fnic_stats *fnic_stats = &fnic->fnic_stats; unsigned long flags; - spinlock_t *io_lock; u64 cmd_trace; unsigned long start_time; unsigned long io_duration_time; + unsigned int hwq = 0; + unsigned int mqtag = 0; + unsigned int tag = 0; /* Decode the cmpl description to get the io_req id */ - fcpio_header_dec(&desc->hdr, &type, &hdr_status, &tag); - fcpio_tag_id_dec(&tag, &id); + fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag); + fcpio_tag_id_dec(&ftag, &id); icmnd_cmpl = &desc->u.icmnd_cmpl; - if (id >= fnic->fnic_max_tag_id) { - shost_printk(KERN_ERR, fnic->lport->host, - "Tag out of range tag %x hdr status = %s\n", - id, fnic_fcpio_status_to_str(hdr_status)); + mqtag = id; + tag = blk_mq_unique_tag_to_tag(mqtag); + hwq = blk_mq_unique_tag_to_hwq(mqtag); + + if (hwq != cq_index) { + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hdr status: %s icmnd completion on the wrong queue\n", + fnic_fcpio_status_to_str(hdr_status)); + } + + if (tag >= fnic->fnic_max_tag_id) { + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hdr status: %s Out of range tag\n", + fnic_fcpio_status_to_str(hdr_status)); return; } + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); sc = scsi_host_find_tag(fnic->lport->host, id); WARN_ON_ONCE(!sc); if (!sc) { atomic64_inc(&fnic_stats->io_stats.sc_null); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); shost_printk(KERN_ERR, fnic->lport->host, "icmnd_cmpl sc is null - " "hdr status = %s tag = 0x%x desc = 0x%p\n", @@ -852,14 +872,19 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, return; } - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); io_req = fnic_priv(sc)->io_req; + if (fnic->sw_copy_wq[hwq].io_req_table[tag] != io_req) { + WARN(1, "%s: %d: hwq: %d mqtag: 0x%x tag: 0x%x io_req tag mismatch\n", + __func__, __LINE__, hwq, mqtag, tag); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return; + } + WARN_ON_ONCE(!io_req); if (!io_req) { atomic64_inc(&fnic_stats->io_stats.ioreq_null); fnic_priv(sc)->flags |= FNIC_IO_REQ_NULL; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); shost_printk(KERN_ERR, fnic->lport->host, "icmnd_cmpl io_req is null - " "hdr status = %s tag = 0x%x sc 0x%p\n", @@ -883,11 +908,11 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, */ fnic_priv(sc)->flags |= FNIC_IO_DONE; fnic_priv(sc)->flags |= FNIC_IO_ABTS_PENDING; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if(FCPIO_ABORTED == hdr_status) fnic_priv(sc)->flags |= FNIC_IO_ABORTED; - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "icmnd_cmpl abts pending " "hdr status = %s tag = 0x%x sc = 0x%p " "scsi_status = %x residual = %d\n", @@ -971,7 +996,11 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, /* Break link with the SCSI command */ fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; fnic_priv(sc)->flags |= FNIC_IO_DONE; + fnic->sw_copy_wq[hwq].io_req_table[tag] = NULL; + + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (hdr_status != FCPIO_SUCCESS) { atomic64_inc(&fnic_stats->io_stats.io_failures); @@ -1005,7 +1034,6 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, /* Call SCSI completion function to complete the IO */ scsi_done(sc); - spin_unlock_irqrestore(io_lock, flags); mempool_free(io_req, fnic->io_req_pool); @@ -1042,7 +1070,7 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, /* fnic_fcpio_itmf_cmpl_handler * Routine to handle itmf completions */ -static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, +static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_index, struct fcpio_fw_req *desc) { u8 type; @@ -1056,51 +1084,73 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; unsigned long flags; - spinlock_t *io_lock; unsigned long start_time; + unsigned int hwq = cq_index; + unsigned int mqtag; unsigned int tag; fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag); fcpio_tag_id_dec(&ftag, &id); - tag = id & FNIC_TAG_MASK; - if (tag == fnic->fnic_max_tag_id) { - if (!(id & FNIC_TAG_DEV_RST)) { - shost_printk(KERN_ERR, fnic->lport->host, - "Tag out of range id 0x%x hdr status = %s\n", - id, fnic_fcpio_status_to_str(hdr_status)); - return; - } - } else if (tag > fnic->fnic_max_tag_id) { - shost_printk(KERN_ERR, fnic->lport->host, - "Tag out of range tag 0x%x hdr status = %s\n", - tag, fnic_fcpio_status_to_str(hdr_status)); + mqtag = id & FNIC_TAG_MASK; + tag = blk_mq_unique_tag_to_tag(id & FNIC_TAG_MASK); + hwq = blk_mq_unique_tag_to_hwq(id & FNIC_TAG_MASK); + + if (hwq != cq_index) { + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hdr status: %s ITMF completion on the wrong queue\n", + fnic_fcpio_status_to_str(hdr_status)); + } + + if (tag > fnic->fnic_max_tag_id) { + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hdr status: %s Tag out of range\n", + fnic_fcpio_status_to_str(hdr_status)); + return; + } else if ((tag == fnic->fnic_max_tag_id) && !(id & FNIC_TAG_DEV_RST)) { + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hdr status: %s Tag out of range\n", + fnic_fcpio_status_to_str(hdr_status)); return; } - if ((tag == fnic->fnic_max_tag_id) && (id & FNIC_TAG_DEV_RST)) { - sc = fnic->sgreset_sc; - io_lock = &fnic->sgreset_lock; + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + + /* If it is sg3utils allocated SC then tag_id + * is max_tag_id and SC is retrieved from io_req + */ + if ((mqtag == fnic->fnic_max_tag_id) && (id & FNIC_TAG_DEV_RST)) { + io_req = fnic->sw_copy_wq[hwq].io_req_table[tag]; + if (io_req) + sc = io_req->sc; } else { sc = scsi_host_find_tag(fnic->lport->host, id & FNIC_TAG_MASK); - io_lock = fnic_io_lock_hash(fnic, sc); } WARN_ON_ONCE(!sc); if (!sc) { atomic64_inc(&fnic_stats->io_stats.sc_null); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); shost_printk(KERN_ERR, fnic->lport->host, "itmf_cmpl sc is null - hdr status = %s tag = 0x%x\n", fnic_fcpio_status_to_str(hdr_status), tag); return; } - spin_lock_irqsave(io_lock, flags); io_req = fnic_priv(sc)->io_req; WARN_ON_ONCE(!io_req); if (!io_req) { atomic64_inc(&fnic_stats->io_stats.ioreq_null); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_REQ_NULL; shost_printk(KERN_ERR, fnic->lport->host, "itmf_cmpl io_req is null - " @@ -1113,17 +1163,22 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, if ((id & FNIC_TAG_ABORT) && (id & FNIC_TAG_DEV_RST)) { /* Abort and terminate completion of device reset req */ /* REVISIT : Add asserts about various flags */ - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "dev reset abts cmpl recd. id %x status %s\n", - id, fnic_fcpio_status_to_str(hdr_status)); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Abt/term completion received\n", + hwq, mqtag, tag, + fnic_fcpio_status_to_str(hdr_status)); fnic_priv(sc)->state = FNIC_IOREQ_ABTS_COMPLETE; fnic_priv(sc)->abts_status = hdr_status; fnic_priv(sc)->flags |= FNIC_DEV_RST_DONE; if (io_req->abts_done) complete(io_req->abts_done); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } else if (id & FNIC_TAG_ABORT) { /* Completion of abort cmd */ + shost_printk(KERN_DEBUG, fnic->lport->host, + "hwq: %d mqtag: 0x%x tag: 0x%x Abort header status: %s\n", + hwq, mqtag, tag, + fnic_fcpio_status_to_str(hdr_status)); switch (hdr_status) { case FCPIO_SUCCESS: break; @@ -1135,7 +1190,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, &term_stats->terminate_fw_timeouts); break; case FCPIO_ITMF_REJECTED: - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, "abort reject recd. id %d\n", (int)(id & FNIC_TAG_MASK)); break; @@ -1156,7 +1211,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, } if (fnic_priv(sc)->state != FNIC_IOREQ_ABTS_PENDING) { /* This is a late completion. Ignore it */ - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return; } @@ -1170,7 +1225,7 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, if (!(fnic_priv(sc)->flags & (FNIC_IO_ABORTED | FNIC_IO_DONE))) atomic64_inc(&misc_stats->no_icmnd_itmf_cmpls); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "abts cmpl recd. id %d status %s\n", (int)(id & FNIC_TAG_MASK), fnic_fcpio_status_to_str(hdr_status)); @@ -1182,14 +1237,19 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, */ if (io_req->abts_done) { complete(io_req->abts_done); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + shost_printk(KERN_INFO, fnic->lport->host, + "hwq: %d mqtag: 0x%x tag: 0x%x Waking up abort thread\n", + hwq, mqtag, tag); } else { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "abts cmpl, completing IO\n"); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Completing IO\n", + hwq, mqtag, + tag, fnic_fcpio_status_to_str(hdr_status)); fnic_priv(sc)->io_req = NULL; sc->result = (DID_ERROR << 16); - - spin_unlock_irqrestore(io_lock, flags); + fnic->sw_copy_wq[hwq].io_req_table[tag] = NULL; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_release_ioreq_buf(fnic, io_req, sc); mempool_free(io_req, fnic->io_req_pool); @@ -1213,29 +1273,32 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, } } else if (id & FNIC_TAG_DEV_RST) { /* Completion of device reset */ + shost_printk(KERN_INFO, fnic->lport->host, + "hwq: %d mqtag: 0x%x tag: 0x%x DR hst: %s\n", + hwq, mqtag, + tag, fnic_fcpio_status_to_str(hdr_status)); fnic_priv(sc)->lr_status = hdr_status; if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_priv(sc)->flags |= FNIC_DEV_RST_ABTS_PENDING; FNIC_TRACE(fnic_fcpio_itmf_cmpl_handler, sc->device->host->host_no, id, sc, jiffies_to_msecs(jiffies - start_time), desc, 0, fnic_flags_and_state(sc)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Terminate pending " - "dev reset cmpl recd. id %d status %s\n", - (int)(id & FNIC_TAG_MASK), - fnic_fcpio_status_to_str(hdr_status)); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Terminate pending\n", + hwq, mqtag, + tag, fnic_fcpio_status_to_str(hdr_status)); return; } if (fnic_priv(sc)->flags & FNIC_DEV_RST_TIMED_OUT) { /* Need to wait for terminate completion */ - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); FNIC_TRACE(fnic_fcpio_itmf_cmpl_handler, sc->device->host->host_no, id, sc, jiffies_to_msecs(jiffies - start_time), desc, 0, fnic_flags_and_state(sc)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "dev reset cmpl recd after time out. " "id %d status %s\n", (int)(id & FNIC_TAG_MASK), @@ -1244,19 +1307,19 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, } fnic_priv(sc)->state = FNIC_IOREQ_CMD_COMPLETE; fnic_priv(sc)->flags |= FNIC_DEV_RST_DONE; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "dev reset cmpl recd. id %d status %s\n", - (int)(id & FNIC_TAG_MASK), - fnic_fcpio_status_to_str(hdr_status)); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s DR completion received\n", + hwq, mqtag, + tag, fnic_fcpio_status_to_str(hdr_status)); if (io_req->dr_done) complete(io_req->dr_done); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } else { shost_printk(KERN_ERR, fnic->lport->host, - "Unexpected itmf io state %s tag %x\n", - fnic_ioreq_state_to_str(fnic_priv(sc)->state), id); - spin_unlock_irqrestore(io_lock, flags); + "%s: Unexpected itmf io state: hwq: %d tag 0x%x %s\n", + __func__, hwq, id, fnic_ioreq_state_to_str(fnic_priv(sc)->state)); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } } @@ -1283,17 +1346,19 @@ static int fnic_fcpio_cmpl_handler(struct vnic_dev *vdev, break; } + cq_index -= fnic->copy_wq_base; + switch (desc->hdr.type) { case FCPIO_ACK: /* fw copied copy wq desc to its queue */ fnic_fcpio_ack_handler(fnic, cq_index, desc); break; case FCPIO_ICMND_CMPL: /* fw completed a command */ - fnic_fcpio_icmnd_cmpl_handler(fnic, desc); + fnic_fcpio_icmnd_cmpl_handler(fnic, cq_index, desc); break; case FCPIO_ITMF_CMPL: /* fw completed itmf (abort cmd, lun reset)*/ - fnic_fcpio_itmf_cmpl_handler(fnic, desc); + fnic_fcpio_itmf_cmpl_handler(fnic, cq_index, desc); break; case FCPIO_FLOGI_REG_CMPL: /* fw completed flogi_reg */ @@ -1306,7 +1371,7 @@ static int fnic_fcpio_cmpl_handler(struct vnic_dev *vdev, break; default: - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "firmware completion type %d\n", desc->hdr.type); break; @@ -1319,10 +1384,8 @@ static int fnic_fcpio_cmpl_handler(struct vnic_dev *vdev, * fnic_wq_copy_cmpl_handler * Routine to process wq copy */ -int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do) +int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do, unsigned int cq_index) { - unsigned int wq_work_done = 0; - unsigned int i, cq_index; unsigned int cur_work_done; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; u64 start_jiffies = 0; @@ -1330,44 +1393,51 @@ int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do) u64 delta_jiffies = 0; u64 delta_ms = 0; - for (i = 0; i < fnic->wq_copy_count; i++) { - cq_index = i + fnic->raw_wq_count + fnic->rq_count; - - start_jiffies = jiffies; - cur_work_done = vnic_cq_copy_service(&fnic->cq[cq_index], - fnic_fcpio_cmpl_handler, - copy_work_to_do); - end_jiffies = jiffies; - - wq_work_done += cur_work_done; - delta_jiffies = end_jiffies - start_jiffies; - if (delta_jiffies > - (u64) atomic64_read(&misc_stats->max_isr_jiffies)) { - atomic64_set(&misc_stats->max_isr_jiffies, - delta_jiffies); - delta_ms = jiffies_to_msecs(delta_jiffies); - atomic64_set(&misc_stats->max_isr_time_ms, delta_ms); - atomic64_set(&misc_stats->corr_work_done, - cur_work_done); - } + start_jiffies = jiffies; + cur_work_done = vnic_cq_copy_service(&fnic->cq[cq_index], + fnic_fcpio_cmpl_handler, + copy_work_to_do); + end_jiffies = jiffies; + delta_jiffies = end_jiffies - start_jiffies; + if (delta_jiffies > (u64) atomic64_read(&misc_stats->max_isr_jiffies)) { + atomic64_set(&misc_stats->max_isr_jiffies, delta_jiffies); + delta_ms = jiffies_to_msecs(delta_jiffies); + atomic64_set(&misc_stats->max_isr_time_ms, delta_ms); + atomic64_set(&misc_stats->corr_work_done, cur_work_done); } - return wq_work_done; + + return cur_work_done; } static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data) { - const int tag = scsi_cmd_to_rq(sc)->tag; + struct request *const rq = scsi_cmd_to_rq(sc); struct fnic *fnic = data; struct fnic_io_req *io_req; unsigned long flags = 0; - spinlock_t *io_lock; unsigned long start_time = 0; struct fnic_stats *fnic_stats = &fnic->fnic_stats; + uint16_t hwq = 0; + int tag; + int mqtag; - io_lock = fnic_io_lock_tag(fnic, tag); - spin_lock_irqsave(io_lock, flags); + mqtag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(mqtag); + tag = blk_mq_unique_tag_to_tag(mqtag); + + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + + fnic->sw_copy_wq[hwq].io_req_table[tag] = NULL; io_req = fnic_priv(sc)->io_req; + if (!io_req) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x flags: 0x%x No ioreq. Returning\n", + hwq, mqtag, tag, fnic_priv(sc)->flags); + return true; + } + if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) && !(fnic_priv(sc)->flags & FNIC_DEV_RST_DONE)) { /* @@ -1379,20 +1449,16 @@ static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data) complete(io_req->dr_done); else if (io_req && io_req->abts_done) complete(io_req->abts_done); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } else if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } - if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); - goto cleanup_scsi_cmd; - } fnic_priv(sc)->io_req = NULL; - - spin_unlock_irqrestore(io_lock, flags); + io_req->sc = NULL; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); /* * If there is a scsi_cmnd associated with this io_req, then @@ -1402,23 +1468,16 @@ static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data) fnic_release_ioreq_buf(fnic, io_req, sc); mempool_free(io_req, fnic->io_req_pool); -cleanup_scsi_cmd: sc->result = DID_TRANSPORT_DISRUPTED << 16; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_cleanup_io: tag:0x%x : sc:0x%p duration = %lu DID_TRANSPORT_DISRUPTED\n", - tag, sc, jiffies - start_time); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "mqtag:0x%x tag: 0x%x sc:0x%p duration = %lu DID_TRANSPORT_DISRUPTED\n", + mqtag, tag, sc, (jiffies - start_time)); if (atomic64_read(&fnic->io_cmpl_skip)) atomic64_dec(&fnic->io_cmpl_skip); else atomic64_inc(&fnic_stats->io_stats.io_completions); - /* Complete the command to SCSI */ - if (!(fnic_priv(sc)->flags & FNIC_IO_ISSUED)) - shost_printk(KERN_ERR, fnic->lport->host, - "Calling done for IO not issued to fw: tag:0x%x sc:0x%p\n", - tag, sc); - FNIC_TRACE(fnic_cleanup_io, sc->device->host->host_no, tag, sc, jiffies_to_msecs(jiffies - start_time), @@ -1447,8 +1506,8 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, struct fnic_io_req *io_req; struct scsi_cmnd *sc; unsigned long flags; - spinlock_t *io_lock; unsigned long start_time = 0; + uint16_t hwq; /* get the tag reference */ fcpio_tag_id_dec(&desc->hdr.tag, &id); @@ -1461,8 +1520,8 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, if (!sc) return; - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); + hwq = blk_mq_unique_tag_to_hwq(id); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); /* Get the IO context which this desc refers to */ io_req = fnic_priv(sc)->io_req; @@ -1470,13 +1529,15 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, /* fnic interrupts are turned off by now */ if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto wq_copy_cleanup_scsi_cmd; } fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(id)] = NULL; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); start_time = io_req->start_time; fnic_release_ioreq_buf(fnic, io_req, sc); @@ -1484,7 +1545,7 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, wq_copy_cleanup_scsi_cmd: sc->result = DID_NO_CONNECT << 16; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, "wq_copy_cleanup_handler:" + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "wq_copy_cleanup_handler:" " DID_NO_CONNECT\n"); FNIC_TRACE(fnic_wq_copy_cleanup_handler, @@ -1500,31 +1561,31 @@ wq_copy_cleanup_scsi_cmd: static inline int fnic_queue_abort_io_req(struct fnic *fnic, int tag, u32 task_req, u8 *fc_lun, - struct fnic_io_req *io_req) + struct fnic_io_req *io_req, + unsigned int hwq) { - struct vnic_wq_copy *wq = &fnic->wq_copy[0]; - struct Scsi_Host *host = fnic->lport->host; + struct vnic_wq_copy *wq = &fnic->hw_copy_wq[hwq]; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; unsigned long flags; - spin_lock_irqsave(host->host_lock, flags); + spin_lock_irqsave(&fnic->fnic_lock, flags); if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) { - spin_unlock_irqrestore(host->host_lock, flags); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); return 1; } else atomic_inc(&fnic->in_flight); - spin_unlock_irqrestore(host->host_lock, flags); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); - spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[hwq]) + free_wq_copy_descs(fnic, wq, hwq); if (!vnic_wq_copy_desc_avail(wq)) { - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fnic_queue_abort_io_req: failure: no descriptors\n"); atomic64_inc(&misc_stats->abts_cpwq_alloc_failures); return 1; @@ -1539,7 +1600,7 @@ static inline int fnic_queue_abort_io_req(struct fnic *fnic, int tag, atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs, atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs)); - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); return 0; @@ -1553,33 +1614,36 @@ struct fnic_rport_abort_io_iter_data { static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) { + struct request *const rq = scsi_cmd_to_rq(sc); struct fnic_rport_abort_io_iter_data *iter_data = data; struct fnic *fnic = iter_data->fnic; - int abt_tag = scsi_cmd_to_rq(sc)->tag; + int abt_tag = 0; struct fnic_io_req *io_req; - spinlock_t *io_lock; unsigned long flags; struct reset_stats *reset_stats = &fnic->fnic_stats.reset_stats; struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats; struct scsi_lun fc_lun; enum fnic_ioreq_state old_ioreq_state; + uint16_t hwq = 0; - io_lock = fnic_io_lock_tag(fnic, abt_tag); - spin_lock_irqsave(io_lock, flags); + abt_tag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(abt_tag); + + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (!io_req || io_req->port_id != iter_data->port_id) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) && !(fnic_priv(sc)->flags & FNIC_DEV_RST_ISSUED)) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_rport_exch_reset dev rst not pending sc 0x%p\n", - sc); - spin_unlock_irqrestore(io_lock, flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d abt_tag: 0x%x flags: 0x%x Device reset is not pending\n", + hwq, abt_tag, fnic_priv(sc)->flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } @@ -1588,7 +1652,7 @@ static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) * belongs to rport that went away */ if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } if (io_req->abts_done) { @@ -1612,38 +1676,41 @@ static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) atomic64_inc(&reset_stats->device_reset_terminates); abt_tag |= FNIC_TAG_DEV_RST; } - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fnic_rport_exch_reset dev rst sc 0x%p\n", sc); BUG_ON(io_req->abts_done); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fnic_rport_reset_exch: Issuing abts\n"); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); /* Now queue the abort command to firmware */ int_to_scsilun(sc->device->lun, &fc_lun); if (fnic_queue_abort_io_req(fnic, abt_tag, FCPIO_ITMF_ABT_TASK_TERM, - fc_lun.scsi_lun, io_req)) { + fc_lun.scsi_lun, io_req, hwq)) { /* * Revert the cmd state back to old state, if * it hasn't changed in between. This cmd will get * aborted later by scsi_eh, or cleaned up during * lun reset */ - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d abt_tag: 0x%x flags: 0x%x Queuing abort failed\n", + hwq, abt_tag, fnic_priv(sc)->flags); if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) fnic_priv(sc)->state = old_ioreq_state; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } else { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) fnic_priv(sc)->flags |= FNIC_DEV_RST_TERM_ISSUED; else fnic_priv(sc)->flags |= FNIC_IO_INTERNAL_TERM_ISSUED; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic64_inc(&term_stats->terminates); iter_data->term_cnt++; } @@ -1660,7 +1727,7 @@ static void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id) }; FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, + fnic->lport->host, fnic->fnic_num, "fnic_rport_exch_reset called portid 0x%06x\n", port_id); @@ -1697,9 +1764,8 @@ void fnic_terminate_rport_io(struct fc_rport *rport) return; } fnic = lport_priv(lport); - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, "fnic_terminate_rport_io called" - " wwpn 0x%llx, wwnn0x%llx, rport 0x%p, portid 0x%06x\n", + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, + "wwpn 0x%llx, wwnn0x%llx, rport 0x%p, portid 0x%06x\n", rport->port_name, rport->node_name, rport, rport->port_id); @@ -1721,7 +1787,6 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) struct fnic *fnic; struct fnic_io_req *io_req = NULL; struct fc_rport *rport; - spinlock_t *io_lock; unsigned long flags; unsigned long start_time = 0; int ret = SUCCESS; @@ -1731,8 +1796,10 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) struct abort_stats *abts_stats; struct terminate_stats *term_stats; enum fnic_ioreq_state old_ioreq_state; - const int tag = rq->tag; + int mqtag; unsigned long abt_issued_time; + uint16_t hwq = 0; + DECLARE_COMPLETION_ONSTACK(tm_done); /* Wait for rport to unblock */ @@ -1742,23 +1809,25 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) lp = shost_priv(sc->device->host); fnic = lport_priv(lp); + + spin_lock_irqsave(&fnic->fnic_lock, flags); fnic_stats = &fnic->fnic_stats; abts_stats = &fnic->fnic_stats.abts_stats; term_stats = &fnic->fnic_stats.term_stats; rport = starget_to_rport(scsi_target(sc->device)); - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "Abort Cmd called FCID 0x%x, LUN 0x%llx TAG %x flags %x\n", - rport->port_id, sc->device->lun, tag, fnic_priv(sc)->flags); + mqtag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(mqtag); fnic_priv(sc)->flags = FNIC_NO_FLAGS; if (lp->state != LPORT_ST_READY || !(lp->link_up)) { ret = FAILED; + spin_unlock_irqrestore(&fnic->fnic_lock, flags); goto fnic_abort_cmd_end; } + spin_unlock_irqrestore(&fnic->fnic_lock, flags); /* * Avoid a race between SCSI issuing the abort and the device * completing the command. @@ -1771,18 +1840,17 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) * * .io_req will not be cleared except while holding io_req_lock. */ - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto fnic_abort_cmd_end; } io_req->abts_done = &tm_done; if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto wait_pending; } @@ -1802,8 +1870,9 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) else atomic64_inc(&abts_stats->abort_issued_greater_than_60_sec); - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "CBD Opcode: %02x Abort issued time: %lu msec\n", sc->cmnd[0], abt_issued_time); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, + "CDB Opcode: 0x%02x Abort issued time: %lu msec\n", + sc->cmnd[0], abt_issued_time); /* * Command is still pending, need to abort it * If the firmware completes the command after this point, @@ -1814,7 +1883,7 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) fnic_priv(sc)->state = FNIC_IOREQ_ABTS_PENDING; fnic_priv(sc)->abts_status = FCPIO_INVALID_CODE; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); /* * Check readiness of the remote port. If the path to remote @@ -1831,15 +1900,15 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) /* Now queue the abort command to firmware */ int_to_scsilun(sc->device->lun, &fc_lun); - if (fnic_queue_abort_io_req(fnic, tag, task_req, fc_lun.scsi_lun, - io_req)) { - spin_lock_irqsave(io_lock, flags); + if (fnic_queue_abort_io_req(fnic, mqtag, task_req, fc_lun.scsi_lun, + io_req, hwq)) { + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) fnic_priv(sc)->state = old_ioreq_state; io_req = fnic_priv(sc)->io_req; if (io_req) io_req->abts_done = NULL; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); ret = FAILED; goto fnic_abort_cmd_end; } @@ -1863,12 +1932,12 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) fnic->config.ed_tov)); /* Check the abort status */ - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (!io_req) { atomic64_inc(&fnic_stats->io_stats.ioreq_null); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_REQ_NULL; ret = FAILED; goto fnic_abort_cmd_end; @@ -1877,7 +1946,7 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) /* fw did not complete abort, timed out */ if (fnic_priv(sc)->abts_status == FCPIO_INVALID_CODE) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (task_req == FCPIO_ITMF_ABT_TASK) { atomic64_inc(&abts_stats->abort_drv_timeouts); } else { @@ -1891,9 +1960,9 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) /* IO out of order */ if (!(fnic_priv(sc)->flags & (FNIC_IO_ABORTED | FNIC_IO_DONE))) { - spin_unlock_irqrestore(io_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Issuing Host reset due to out of order IO\n"); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "Issuing host reset due to out of order IO\n"); ret = FAILED; goto fnic_abort_cmd_end; @@ -1907,15 +1976,18 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) * free the io_req if successful. If abort fails, * Device reset will clean the I/O. */ - if (fnic_priv(sc)->abts_status == FCPIO_SUCCESS) { + if (fnic_priv(sc)->abts_status == FCPIO_SUCCESS || + (fnic_priv(sc)->abts_status == FCPIO_ABORTED)) { fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; } else { ret = FAILED; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto fnic_abort_cmd_end; } - spin_unlock_irqrestore(io_lock, flags); + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] = NULL; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_release_ioreq_buf(fnic, io_req, sc); mempool_free(io_req, fnic->io_req_pool); @@ -1930,14 +2002,14 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) atomic64_inc(&fnic_stats->io_stats.io_completions); fnic_abort_cmd_end: - FNIC_TRACE(fnic_abort_cmd, sc->device->host->host_no, tag, sc, + FNIC_TRACE(fnic_abort_cmd, sc->device->host->host_no, mqtag, sc, jiffies_to_msecs(jiffies - start_time), 0, ((u64)sc->cmnd[0] << 32 | (u64)sc->cmnd[2] << 24 | (u64)sc->cmnd[3] << 16 | (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), fnic_flags_and_state(sc)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Returning from abort cmd type %x %s\n", task_req, (ret == SUCCESS) ? "SUCCESS" : "FAILED"); @@ -1948,33 +2020,34 @@ static inline int fnic_queue_dr_io_req(struct fnic *fnic, struct scsi_cmnd *sc, struct fnic_io_req *io_req) { - struct vnic_wq_copy *wq = &fnic->wq_copy[0]; - struct Scsi_Host *host = fnic->lport->host; + struct vnic_wq_copy *wq; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; struct scsi_lun fc_lun; int ret = 0; - unsigned long intr_flags; - unsigned int tag = scsi_cmd_to_rq(sc)->tag; + unsigned long flags; + uint16_t hwq = 0; + uint32_t tag = 0; - if (tag == SCSI_NO_TAG) - tag = io_req->tag; + tag = io_req->tag; + hwq = blk_mq_unique_tag_to_hwq(tag); + wq = &fnic->hw_copy_wq[hwq]; - spin_lock_irqsave(host->host_lock, intr_flags); + spin_lock_irqsave(&fnic->fnic_lock, flags); if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) { - spin_unlock_irqrestore(host->host_lock, intr_flags); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); return FAILED; } else atomic_inc(&fnic->in_flight); - spin_unlock_irqrestore(host->host_lock, intr_flags); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); - spin_lock_irqsave(&fnic->wq_copy_lock[0], intr_flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[hwq]) + free_wq_copy_descs(fnic, wq, hwq); if (!vnic_wq_copy_desc_avail(wq)) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "queue_dr_io_req failure - no descriptors\n"); atomic64_inc(&misc_stats->devrst_cpwq_alloc_failures); ret = -EAGAIN; @@ -1997,7 +2070,7 @@ static inline int fnic_queue_dr_io_req(struct fnic *fnic, atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs)); lr_io_req_end: - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); return ret; @@ -2012,12 +2085,13 @@ struct fnic_pending_aborts_iter_data { static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) { + struct request *const rq = scsi_cmd_to_rq(sc); struct fnic_pending_aborts_iter_data *iter_data = data; struct fnic *fnic = iter_data->fnic; struct scsi_device *lun_dev = iter_data->lun_dev; - int abt_tag = scsi_cmd_to_rq(sc)->tag; + unsigned long abt_tag = 0; + uint16_t hwq = 0; struct fnic_io_req *io_req; - spinlock_t *io_lock; unsigned long flags; struct scsi_lun fc_lun; DECLARE_COMPLETION_ONSTACK(tm_done); @@ -2026,11 +2100,13 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) if (sc == iter_data->lr_sc || sc->device != lun_dev) return true; - io_lock = fnic_io_lock_tag(fnic, abt_tag); - spin_lock_irqsave(io_lock, flags); + abt_tag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(abt_tag); + + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } @@ -2038,20 +2114,19 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) * Found IO that is still pending with firmware and * belongs to the LUN that we are resetting */ - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Found IO in %s on lun\n", fnic_ioreq_state_to_str(fnic_priv(sc)->state)); if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) && (!(fnic_priv(sc)->flags & FNIC_DEV_RST_ISSUED))) { - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "%s dev rst not pending sc 0x%p\n", __func__, - sc); - spin_unlock_irqrestore(io_lock, flags); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "dev rst not pending sc 0x%p\n", sc); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } @@ -2072,35 +2147,37 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) BUG_ON(io_req->abts_done); if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) { - abt_tag |= FNIC_TAG_DEV_RST; - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "%s: dev rst sc 0x%p\n", __func__, sc); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "dev rst sc 0x%p\n", sc); } fnic_priv(sc)->abts_status = FCPIO_INVALID_CODE; io_req->abts_done = &tm_done; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); /* Now queue the abort command to firmware */ int_to_scsilun(sc->device->lun, &fc_lun); if (fnic_queue_abort_io_req(fnic, abt_tag, FCPIO_ITMF_ABT_TASK_TERM, - fc_lun.scsi_lun, io_req)) { - spin_lock_irqsave(io_lock, flags); + fc_lun.scsi_lun, io_req, hwq)) { + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (io_req) io_req->abts_done = NULL; if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) fnic_priv(sc)->state = old_ioreq_state; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); iter_data->ret = FAILED; + FNIC_SCSI_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, + "hwq: %d abt_tag: 0x%lx Abort could not be queued\n", + hwq, abt_tag); return false; } else { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) fnic_priv(sc)->flags |= FNIC_DEV_RST_TERM_ISSUED; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } fnic_priv(sc)->flags |= FNIC_IO_INTERNAL_TERM_ISSUED; @@ -2108,10 +2185,10 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) (fnic->config.ed_tov)); /* Recheck cmd state to check if it is now aborted */ - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_REQ_NULL; return true; } @@ -2120,7 +2197,7 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) /* if abort is still pending with fw, fail */ if (fnic_priv(sc)->abts_status == FCPIO_INVALID_CODE) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_DONE; iter_data->ret = FAILED; return false; @@ -2128,9 +2205,11 @@ static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) fnic_priv(sc)->state = FNIC_IOREQ_ABTS_COMPLETE; /* original sc used for lr is handled by dev reset code */ - if (sc != iter_data->lr_sc) + if (sc != iter_data->lr_sc) { fnic_priv(sc)->io_req = NULL; - spin_unlock_irqrestore(io_lock, flags); + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(abt_tag)] = NULL; + } + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); /* original sc used for lr is handled by dev reset code */ if (sc != iter_data->lr_sc) { @@ -2182,8 +2261,8 @@ static int fnic_clean_pending_aborts(struct fnic *fnic, ret = 1; clean_pending_aborts_end: - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "%s: exit status: %d\n", __func__, ret); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "exit status: %d\n", ret); return ret; } @@ -2201,15 +2280,15 @@ int fnic_device_reset(struct scsi_cmnd *sc) struct fc_rport *rport; int status; int ret = FAILED; - spinlock_t *io_lock; unsigned long flags; unsigned long start_time = 0; struct scsi_lun fc_lun; struct fnic_stats *fnic_stats; struct reset_stats *reset_stats; - int tag = rq->tag; + int mqtag = rq->tag; DECLARE_COMPLETION_ONSTACK(tm_done); bool new_sc = 0; + uint16_t hwq = 0; /* Wait for rport to unblock */ fc_block_scsi_eh(sc); @@ -2224,9 +2303,10 @@ int fnic_device_reset(struct scsi_cmnd *sc) atomic64_inc(&reset_stats->device_resets); rport = starget_to_rport(scsi_target(sc->device)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Device reset called FCID 0x%x, LUN 0x%llx sc 0x%p\n", - rport->port_id, sc->device->lun, sc); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, + "fcid: 0x%x lun: 0x%llx hwq: %d mqtag: 0x%x flags: 0x%x Device reset\n", + rport->port_id, sc->device->lun, hwq, mqtag, + fnic_priv(sc)->flags); if (lp->state != LPORT_ST_READY || !(lp->link_up)) goto fnic_device_reset_end; @@ -2239,7 +2319,7 @@ int fnic_device_reset(struct scsi_cmnd *sc) fnic_priv(sc)->flags = FNIC_DEVICE_RESET; - if (unlikely(tag < 0)) { + if (unlikely(mqtag < 0)) { /* * For device reset issued through sg3utils, we let * only one LUN_RESET to go through and use a special @@ -2248,17 +2328,14 @@ int fnic_device_reset(struct scsi_cmnd *sc) * allocated by mid layer. */ mutex_lock(&fnic->sgreset_mutex); - tag = fnic->fnic_max_tag_id; + mqtag = fnic->fnic_max_tag_id; new_sc = 1; - fnic->sgreset_sc = sc; - io_lock = &fnic->sgreset_lock; - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "fcid: 0x%x lun: 0x%llx flags: 0x%x tag: 0x%x Issuing sgreset\n", - rport->port_id, sc->device->lun, fnic_priv(sc)->flags, tag); - } else - io_lock = fnic_io_lock_hash(fnic, sc); + } else { + mqtag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(mqtag); + } - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; /* @@ -2268,36 +2345,43 @@ int fnic_device_reset(struct scsi_cmnd *sc) if (!io_req) { io_req = mempool_alloc(fnic->io_req_pool, GFP_ATOMIC); if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto fnic_device_reset_end; } memset(io_req, 0, sizeof(*io_req)); io_req->port_id = rport->port_id; - io_req->tag = tag; - io_req->sc = sc; + io_req->tag = mqtag; fnic_priv(sc)->io_req = io_req; + io_req->sc = sc; + + if (fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] != NULL) + WARN(1, "fnic<%d>: %s: tag 0x%x already exists\n", + fnic->fnic_num, __func__, blk_mq_unique_tag_to_tag(mqtag)); + + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] = + io_req; } io_req->dr_done = &tm_done; fnic_priv(sc)->state = FNIC_IOREQ_CMD_PENDING; fnic_priv(sc)->lr_status = FCPIO_INVALID_CODE; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, "TAG %x\n", tag); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "TAG %x\n", mqtag); /* * issue the device reset, if enqueue failed, clean up the ioreq * and break assoc with scsi cmd */ if (fnic_queue_dr_io_req(fnic, sc, io_req)) { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (io_req) io_req->dr_done = NULL; goto fnic_device_reset_clean; } - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); fnic_priv(sc)->flags |= FNIC_DEV_RST_ISSUED; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); /* * Wait on the local completion for LUN reset. The io_req may be @@ -2306,12 +2390,12 @@ int fnic_device_reset(struct scsi_cmnd *sc) wait_for_completion_timeout(&tm_done, msecs_to_jiffies(FNIC_LUN_RESET_TIMEOUT)); - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "io_req is null tag 0x%x sc 0x%p\n", tag, sc); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, + "io_req is null mqtag 0x%x sc 0x%p\n", mqtag, sc); goto fnic_device_reset_end; } io_req->dr_done = NULL; @@ -2324,44 +2408,44 @@ int fnic_device_reset(struct scsi_cmnd *sc) */ if (status == FCPIO_INVALID_CODE) { atomic64_inc(&reset_stats->device_reset_timeouts); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Device reset timed out\n"); fnic_priv(sc)->flags |= FNIC_DEV_RST_TIMED_OUT; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); int_to_scsilun(sc->device->lun, &fc_lun); /* * Issue abort and terminate on device reset request. * If q'ing of terminate fails, retry it after a delay. */ while (1) { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); if (fnic_priv(sc)->flags & FNIC_DEV_RST_TERM_ISSUED) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); break; } - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (fnic_queue_abort_io_req(fnic, - tag | FNIC_TAG_DEV_RST, + mqtag | FNIC_TAG_DEV_RST, FCPIO_ITMF_ABT_TASK_TERM, - fc_lun.scsi_lun, io_req)) { + fc_lun.scsi_lun, io_req, hwq)) { wait_for_completion_timeout(&tm_done, msecs_to_jiffies(FNIC_ABT_TERM_DELAY_TIMEOUT)); } else { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); fnic_priv(sc)->flags |= FNIC_DEV_RST_TERM_ISSUED; fnic_priv(sc)->state = FNIC_IOREQ_ABTS_PENDING; io_req->abts_done = &tm_done; - spin_unlock_irqrestore(io_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Abort and terminate issued on Device reset " - "tag 0x%x sc 0x%p\n", tag, sc); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, + "Abort and terminate issued on Device reset mqtag 0x%x sc 0x%p\n", + mqtag, sc); break; } } while (1) { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); if (!(fnic_priv(sc)->flags & FNIC_DEV_RST_DONE)) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); wait_for_completion_timeout(&tm_done, msecs_to_jiffies(FNIC_LUN_RESET_TIMEOUT)); break; @@ -2372,14 +2456,14 @@ int fnic_device_reset(struct scsi_cmnd *sc) } } } else { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } /* Completed, but not successful, clean up the io_req, return fail */ if (status != FCPIO_SUCCESS) { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, + fnic->lport->host, fnic->fnic_num, "Device reset completed - failed\n"); io_req = fnic_priv(sc)->io_req; goto fnic_device_reset_clean; @@ -2393,26 +2477,29 @@ int fnic_device_reset(struct scsi_cmnd *sc) * succeeds */ if (fnic_clean_pending_aborts(fnic, sc, new_sc)) { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Device reset failed" " since could not abort all IOs\n"); goto fnic_device_reset_clean; } /* Clean lun reset command */ - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (io_req) /* Completed, and successful */ ret = SUCCESS; fnic_device_reset_clean: - if (io_req) + if (io_req) { fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(io_req->tag)] = NULL; + } - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (io_req) { start_time = io_req->start_time; @@ -2433,7 +2520,7 @@ fnic_device_reset_end: mutex_unlock(&fnic->sgreset_mutex); } - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "Returning from device reset %s\n", (ret == SUCCESS) ? "SUCCESS" : "FAILED"); @@ -2456,8 +2543,8 @@ int fnic_reset(struct Scsi_Host *shost) fnic = lport_priv(lp); reset_stats = &fnic->fnic_stats.reset_stats; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_reset called\n"); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "Issuing fnic reset\n"); atomic64_inc(&reset_stats->fnic_resets); @@ -2467,10 +2554,9 @@ int fnic_reset(struct Scsi_Host *shost) */ ret = fc_lport_reset(lp); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Returning from fnic reset %s\n", - (ret == 0) ? - "SUCCESS" : "FAILED"); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "Returning from fnic reset with: %s\n", + (ret == 0) ? "SUCCESS" : "FAILED"); if (ret == 0) atomic64_inc(&reset_stats->fnic_reset_completions); @@ -2503,7 +2589,7 @@ int fnic_host_reset(struct scsi_cmnd *sc) fnic->internal_reset_inprogress = true; } else { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "host reset in progress skipping another host reset\n"); return SUCCESS; } @@ -2578,7 +2664,7 @@ retry_fw_reset: spin_lock_irqsave(&fnic->fnic_lock, flags); fnic->remove_wait = NULL; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, "fnic_scsi_abort_io %s\n", (fnic->state == FNIC_IN_ETH_MODE) ? "SUCCESS" : "FAILED"); @@ -2652,12 +2738,17 @@ call_fc_exch_mgr_reset: static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data) { + struct request *const rq = scsi_cmd_to_rq(sc); struct fnic_pending_aborts_iter_data *iter_data = data; struct fnic *fnic = iter_data->fnic; int cmd_state; struct fnic_io_req *io_req; - spinlock_t *io_lock; unsigned long flags; + uint16_t hwq = 0; + int tag; + + tag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(tag); /* * ignore this lun reset cmd or cmds that do not belong to @@ -2668,12 +2759,11 @@ static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data) if (iter_data->lun_dev && sc->device != iter_data->lun_dev) return true; - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); io_req = fnic_priv(sc)->io_req; if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return true; } @@ -2681,11 +2771,12 @@ static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data) * Found IO that is still pending with firmware and * belongs to the LUN that we are resetting */ - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "Found IO in %s on lun\n", - fnic_ioreq_state_to_str(fnic_priv(sc)->state)); + FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, + "hwq: %d tag: 0x%x Found IO in state: %s on lun\n", + hwq, tag, + fnic_ioreq_state_to_str(fnic_priv(sc)->state)); cmd_state = fnic_priv(sc)->state; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (cmd_state == FNIC_IOREQ_ABTS_PENDING) iter_data->ret = 1; diff --git a/drivers/scsi/fnic/fnic_stats.h b/drivers/scsi/fnic/fnic_stats.h index bdf639eef8..9d7f98c452 100644 --- a/drivers/scsi/fnic/fnic_stats.h +++ b/drivers/scsi/fnic/fnic_stats.h @@ -2,6 +2,7 @@ /* Copyright 2013 Cisco Systems, Inc. All rights reserved. */ #ifndef _FNIC_STATS_H_ #define _FNIC_STATS_H_ +#define FNIC_MQ_MAX_QUEUES 64 struct stats_timestamps { struct timespec64 last_reset_time; @@ -26,6 +27,7 @@ struct io_path_stats { atomic64_t io_btw_10000_to_30000_msec; atomic64_t io_greater_than_30000_msec; atomic64_t current_max_io_time; + atomic64_t ios[FNIC_MQ_MAX_QUEUES]; }; struct abort_stats { @@ -103,6 +105,7 @@ struct misc_stats { atomic64_t rport_not_ready; atomic64_t frame_errors; atomic64_t current_port_speed; + atomic64_t intx_dummy; }; struct fnic_stats { diff --git a/drivers/scsi/fnic/fnic_trace.c b/drivers/scsi/fnic/fnic_trace.c index be0d7c57b2..aaa4ea02fb 100644 --- a/drivers/scsi/fnic/fnic_trace.c +++ b/drivers/scsi/fnic/fnic_trace.c @@ -204,6 +204,7 @@ int fnic_get_stats_data(struct stats_debug_info *debug, int len = 0; int buf_size = debug->buf_size; struct timespec64 val1, val2; + int i = 0; ktime_get_real_ts64(&val1); len = scnprintf(debug->debug_buffer + len, buf_size - len, @@ -267,6 +268,16 @@ int fnic_get_stats_data(struct stats_debug_info *debug, (u64)atomic64_read(&stats->io_stats.io_greater_than_30000_msec)); len += scnprintf(debug->debug_buffer + len, buf_size - len, + "------------------------------------------\n" + "\t\tIO Queues and cumulative IOs\n" + "------------------------------------------\n"); + + for (i = 0; i < FNIC_MQ_MAX_QUEUES; i++) { + len += scnprintf(debug->debug_buffer + len, buf_size - len, + "Q:%d -> %lld\n", i, (u64)atomic64_read(&stats->io_stats.ios[i])); + } + + len += scnprintf(debug->debug_buffer + len, buf_size - len, "\nCurrent Max IO time : %lld\n", (u64)atomic64_read(&stats->io_stats.current_max_io_time)); diff --git a/drivers/scsi/fnic/vnic_dev.c b/drivers/scsi/fnic/vnic_dev.c index 3e5b437c04..e0b173cc9d 100644 --- a/drivers/scsi/fnic/vnic_dev.c +++ b/drivers/scsi/fnic/vnic_dev.c @@ -143,6 +143,10 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev, vdev->res[type].vaddr = (char __iomem *)bar->vaddr + bar_offset; } + pr_info("res_type_wq: %d res_type_rq: %d res_type_cq: %d res_type_intr_ctrl: %d\n", + vdev->res[RES_TYPE_WQ].count, vdev->res[RES_TYPE_RQ].count, + vdev->res[RES_TYPE_CQ].count, vdev->res[RES_TYPE_INTR_CTRL].count); + return 0; } diff --git a/drivers/scsi/fnic/vnic_scsi.h b/drivers/scsi/fnic/vnic_scsi.h index 4e12f7b32d..f715f7942b 100644 --- a/drivers/scsi/fnic/vnic_scsi.h +++ b/drivers/scsi/fnic/vnic_scsi.h @@ -26,7 +26,7 @@ #define VNIC_FNIC_RATOV_MAX 255000 #define VNIC_FNIC_MAXDATAFIELDSIZE_MIN 256 -#define VNIC_FNIC_MAXDATAFIELDSIZE_MAX 2112 +#define VNIC_FNIC_MAXDATAFIELDSIZE_MAX 2048 #define VNIC_FNIC_FLOGI_RETRIES_MIN 0 #define VNIC_FNIC_FLOGI_RETRIES_MAX 0xffffffff @@ -55,7 +55,7 @@ #define VNIC_FNIC_PORT_DOWN_IO_RETRIES_MAX 255 #define VNIC_FNIC_LUNS_PER_TARGET_MIN 1 -#define VNIC_FNIC_LUNS_PER_TARGET_MAX 1024 +#define VNIC_FNIC_LUNS_PER_TARGET_MAX 4096 /* Device-specific region: scsi configuration */ struct vnic_fc_config { @@ -79,10 +79,19 @@ struct vnic_fc_config { u16 ra_tov; u16 intr_timer; u8 intr_timer_type; + u8 intr_mode; + u8 lun_queue_depth; + u8 io_timeout_retry; + u16 wq_copy_count; }; #define VFCF_FCP_SEQ_LVL_ERR 0x1 /* Enable FCP-2 Error Recovery */ #define VFCF_PERBI 0x2 /* persistent binding info available */ #define VFCF_FIP_CAPABLE 0x4 /* firmware can handle FIP */ +#define VFCF_FC_INITIATOR 0x20 /* FC Initiator Mode */ +#define VFCF_FC_TARGET 0x40 /* FC Target Mode */ +#define VFCF_FC_NVME_INITIATOR 0x80 /* FC-NVMe Initiator Mode */ +#define VFCF_FC_NVME_TARGET 0x100 /* FC-NVMe Target Mode */ + #endif /* _VNIC_SCSI_H_ */ |