summaryrefslogtreecommitdiffstats
path: root/drivers/misc/bcm-vk
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:49:45 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 18:49:45 +0000
commit2c3c1048746a4622d8c89a29670120dc8fab93c4 (patch)
tree848558de17fb3008cdf4d861b01ac7781903ce39 /drivers/misc/bcm-vk
parentInitial commit. (diff)
downloadlinux-2c3c1048746a4622d8c89a29670120dc8fab93c4.tar.xz
linux-2c3c1048746a4622d8c89a29670120dc8fab93c4.zip
Adding upstream version 6.1.76.upstream/6.1.76
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'drivers/misc/bcm-vk')
-rw-r--r--drivers/misc/bcm-vk/Kconfig29
-rw-r--r--drivers/misc/bcm-vk/Makefile12
-rw-r--r--drivers/misc/bcm-vk/bcm_vk.h549
-rw-r--r--drivers/misc/bcm-vk/bcm_vk_dev.c1652
-rw-r--r--drivers/misc/bcm-vk/bcm_vk_msg.c1352
-rw-r--r--drivers/misc/bcm-vk/bcm_vk_msg.h163
-rw-r--r--drivers/misc/bcm-vk/bcm_vk_sg.c275
-rw-r--r--drivers/misc/bcm-vk/bcm_vk_sg.h61
-rw-r--r--drivers/misc/bcm-vk/bcm_vk_tty.c339
9 files changed, 4432 insertions, 0 deletions
diff --git a/drivers/misc/bcm-vk/Kconfig b/drivers/misc/bcm-vk/Kconfig
new file mode 100644
index 000000000..68a972772
--- /dev/null
+++ b/drivers/misc/bcm-vk/Kconfig
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Broadcom VK device
+#
+config BCM_VK
+ tristate "Support for Broadcom VK Accelerators"
+ depends on PCI_MSI
+ help
+ Select this option to enable support for Broadcom
+ VK Accelerators. VK is used for performing
+ multiple specific offload processing tasks in parallel.
+ Such offload tasks assist in such operations as video
+ transcoding, compression, and crypto tasks.
+ This driver enables userspace programs to access these
+ accelerators via /dev/bcm-vk.N devices.
+
+ If unsure, say N.
+
+config BCM_VK_TTY
+ bool "Enable tty ports on a Broadcom VK Accelerator device"
+ depends on TTY
+ depends on BCM_VK
+ help
+ Select this option to enable tty support to allow console
+ access to Broadcom VK Accelerator cards from host.
+
+ Device node will in the form /dev/bcm-vk.x_ttyVKy where:
+ x is the instance of the VK card
+ y is the tty device number on the VK card.
diff --git a/drivers/misc/bcm-vk/Makefile b/drivers/misc/bcm-vk/Makefile
new file mode 100644
index 000000000..1df2ebe85
--- /dev/null
+++ b/drivers/misc/bcm-vk/Makefile
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for Broadcom VK driver
+#
+
+obj-$(CONFIG_BCM_VK) += bcm_vk.o
+bcm_vk-objs := \
+ bcm_vk_dev.o \
+ bcm_vk_msg.o \
+ bcm_vk_sg.o
+
+bcm_vk-$(CONFIG_BCM_VK_TTY) += bcm_vk_tty.o
diff --git a/drivers/misc/bcm-vk/bcm_vk.h b/drivers/misc/bcm-vk/bcm_vk.h
new file mode 100644
index 000000000..25d51222e
--- /dev/null
+++ b/drivers/misc/bcm-vk/bcm_vk.h
@@ -0,0 +1,549 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#ifndef BCM_VK_H
+#define BCM_VK_H
+
+#include <linux/atomic.h>
+#include <linux/firmware.h>
+#include <linux/irq.h>
+#include <linux/kref.h>
+#include <linux/miscdevice.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/poll.h>
+#include <linux/sched/signal.h>
+#include <linux/tty.h>
+#include <linux/uaccess.h>
+#include <uapi/linux/misc/bcm_vk.h>
+
+#include "bcm_vk_msg.h"
+
+#define DRV_MODULE_NAME "bcm-vk"
+
+/*
+ * Load Image is completed in two stages:
+ *
+ * 1) When the VK device boot-up, M7 CPU runs and executes the BootROM.
+ * The Secure Boot Loader (SBL) as part of the BootROM will run
+ * to open up ITCM for host to push BOOT1 image.
+ * SBL will authenticate the image before jumping to BOOT1 image.
+ *
+ * 2) Because BOOT1 image is a secured image, we also called it the
+ * Secure Boot Image (SBI). At second stage, SBI will initialize DDR
+ * and wait for host to push BOOT2 image to DDR.
+ * SBI will authenticate the image before jumping to BOOT2 image.
+ *
+ */
+/* Location of registers of interest in BAR0 */
+
+/* Request register for Secure Boot Loader (SBL) download */
+#define BAR_CODEPUSH_SBL 0x400
+/* Start of ITCM */
+#define CODEPUSH_BOOT1_ENTRY 0x00400000
+#define CODEPUSH_MASK 0xfffff000
+#define CODEPUSH_BOOTSTART BIT(0)
+
+/* Boot Status register */
+#define BAR_BOOT_STATUS 0x404
+
+#define SRAM_OPEN BIT(16)
+#define DDR_OPEN BIT(17)
+
+/* Firmware loader progress status definitions */
+#define FW_LOADER_ACK_SEND_MORE_DATA BIT(18)
+#define FW_LOADER_ACK_IN_PROGRESS BIT(19)
+#define FW_LOADER_ACK_RCVD_ALL_DATA BIT(20)
+
+/* Boot1/2 is running in standalone mode */
+#define BOOT_STDALONE_RUNNING BIT(21)
+
+/* definitions for boot status register */
+#define BOOT_STATE_MASK (0xffffffff & \
+ ~(FW_LOADER_ACK_SEND_MORE_DATA | \
+ FW_LOADER_ACK_IN_PROGRESS | \
+ BOOT_STDALONE_RUNNING))
+
+#define BOOT_ERR_SHIFT 4
+#define BOOT_ERR_MASK (0xf << BOOT_ERR_SHIFT)
+#define BOOT_PROG_MASK 0xf
+
+#define BROM_STATUS_NOT_RUN 0x2
+#define BROM_NOT_RUN (SRAM_OPEN | BROM_STATUS_NOT_RUN)
+#define BROM_STATUS_COMPLETE 0x6
+#define BROM_RUNNING (SRAM_OPEN | BROM_STATUS_COMPLETE)
+#define BOOT1_STATUS_COMPLETE 0x6
+#define BOOT1_RUNNING (DDR_OPEN | BOOT1_STATUS_COMPLETE)
+#define BOOT2_STATUS_COMPLETE 0x6
+#define BOOT2_RUNNING (FW_LOADER_ACK_RCVD_ALL_DATA | \
+ BOOT2_STATUS_COMPLETE)
+
+/* Boot request for Secure Boot Image (SBI) */
+#define BAR_CODEPUSH_SBI 0x408
+/* 64M mapped to BAR2 */
+#define CODEPUSH_BOOT2_ENTRY 0x60000000
+
+#define BAR_CARD_STATUS 0x410
+/* CARD_STATUS definitions */
+#define CARD_STATUS_TTYVK0_READY BIT(0)
+#define CARD_STATUS_TTYVK1_READY BIT(1)
+
+#define BAR_BOOT1_STDALONE_PROGRESS 0x420
+#define BOOT1_STDALONE_SUCCESS (BIT(13) | BIT(14))
+#define BOOT1_STDALONE_PROGRESS_MASK BOOT1_STDALONE_SUCCESS
+
+#define BAR_METADATA_VERSION 0x440
+#define BAR_OS_UPTIME 0x444
+#define BAR_CHIP_ID 0x448
+#define MAJOR_SOC_REV(_chip_id) (((_chip_id) >> 20) & 0xf)
+
+#define BAR_CARD_TEMPERATURE 0x45c
+/* defines for all temperature sensor */
+#define BCM_VK_TEMP_FIELD_MASK 0xff
+#define BCM_VK_CPU_TEMP_SHIFT 0
+#define BCM_VK_DDR0_TEMP_SHIFT 8
+#define BCM_VK_DDR1_TEMP_SHIFT 16
+
+#define BAR_CARD_VOLTAGE 0x460
+/* defines for voltage rail conversion */
+#define BCM_VK_VOLT_RAIL_MASK 0xffff
+#define BCM_VK_3P3_VOLT_REG_SHIFT 16
+
+#define BAR_CARD_ERR_LOG 0x464
+/* Error log register bit definition - register for error alerts */
+#define ERR_LOG_UECC BIT(0)
+#define ERR_LOG_SSIM_BUSY BIT(1)
+#define ERR_LOG_AFBC_BUSY BIT(2)
+#define ERR_LOG_HIGH_TEMP_ERR BIT(3)
+#define ERR_LOG_WDOG_TIMEOUT BIT(4)
+#define ERR_LOG_SYS_FAULT BIT(5)
+#define ERR_LOG_RAMDUMP BIT(6)
+#define ERR_LOG_COP_WDOG_TIMEOUT BIT(7)
+/* warnings */
+#define ERR_LOG_MEM_ALLOC_FAIL BIT(8)
+#define ERR_LOG_LOW_TEMP_WARN BIT(9)
+#define ERR_LOG_ECC BIT(10)
+#define ERR_LOG_IPC_DWN BIT(11)
+
+/* Alert bit definitions detectd on host */
+#define ERR_LOG_HOST_INTF_V_FAIL BIT(13)
+#define ERR_LOG_HOST_HB_FAIL BIT(14)
+#define ERR_LOG_HOST_PCIE_DWN BIT(15)
+
+#define BAR_CARD_ERR_MEM 0x468
+/* defines for mem err, all fields have same width */
+#define BCM_VK_MEM_ERR_FIELD_MASK 0xff
+#define BCM_VK_ECC_MEM_ERR_SHIFT 0
+#define BCM_VK_UECC_MEM_ERR_SHIFT 8
+/* threshold of event occurrence and logs start to come out */
+#define BCM_VK_ECC_THRESHOLD 10
+#define BCM_VK_UECC_THRESHOLD 1
+
+#define BAR_CARD_PWR_AND_THRE 0x46c
+/* defines for power and temp threshold, all fields have same width */
+#define BCM_VK_PWR_AND_THRE_FIELD_MASK 0xff
+#define BCM_VK_LOW_TEMP_THRE_SHIFT 0
+#define BCM_VK_HIGH_TEMP_THRE_SHIFT 8
+#define BCM_VK_PWR_STATE_SHIFT 16
+
+#define BAR_CARD_STATIC_INFO 0x470
+
+#define BAR_INTF_VER 0x47c
+#define BAR_INTF_VER_MAJOR_SHIFT 16
+#define BAR_INTF_VER_MASK 0xffff
+/*
+ * major and minor semantic version numbers supported
+ * Please update as required on interface changes
+ */
+#define SEMANTIC_MAJOR 1
+#define SEMANTIC_MINOR 0
+
+/*
+ * first door bell reg, ie for queue = 0. Only need the first one, as
+ * we will use the queue number to derive the others
+ */
+#define VK_BAR0_REGSEG_DB_BASE 0x484
+#define VK_BAR0_REGSEG_DB_REG_GAP 8 /*
+ * DB register gap,
+ * DB1 at 0x48c and DB2 at 0x494
+ */
+
+/* reset register and specific values */
+#define VK_BAR0_RESET_DB_NUM 3
+#define VK_BAR0_RESET_DB_SOFT 0xffffffff
+#define VK_BAR0_RESET_DB_HARD 0xfffffffd
+#define VK_BAR0_RESET_RAMPDUMP 0xa0000000
+
+#define VK_BAR0_Q_DB_BASE(q_num) (VK_BAR0_REGSEG_DB_BASE + \
+ ((q_num) * VK_BAR0_REGSEG_DB_REG_GAP))
+#define VK_BAR0_RESET_DB_BASE (VK_BAR0_REGSEG_DB_BASE + \
+ (VK_BAR0_RESET_DB_NUM * VK_BAR0_REGSEG_DB_REG_GAP))
+
+#define BAR_BOOTSRC_SELECT 0xc78
+/* BOOTSRC definitions */
+#define BOOTSRC_SOFT_ENABLE BIT(14)
+
+/* Card OS Firmware version size */
+#define BAR_FIRMWARE_TAG_SIZE 50
+#define FIRMWARE_STATUS_PRE_INIT_DONE 0x1f
+
+/* VK MSG_ID defines */
+#define VK_MSG_ID_BITMAP_SIZE 4096
+#define VK_MSG_ID_BITMAP_MASK (VK_MSG_ID_BITMAP_SIZE - 1)
+#define VK_MSG_ID_OVERFLOW 0xffff
+
+/*
+ * BAR1
+ */
+
+/* BAR1 message q definition */
+
+/* indicate if msgq ctrl in BAR1 is populated */
+#define VK_BAR1_MSGQ_DEF_RDY 0x60c0
+/* ready marker value for the above location, normal boot2 */
+#define VK_BAR1_MSGQ_RDY_MARKER 0xbeefcafe
+/* ready marker value for the above location, normal boot2 */
+#define VK_BAR1_DIAG_RDY_MARKER 0xdeadcafe
+/* number of msgqs in BAR1 */
+#define VK_BAR1_MSGQ_NR 0x60c4
+/* BAR1 queue control structure offset */
+#define VK_BAR1_MSGQ_CTRL_OFF 0x60c8
+
+/* BAR1 ucode and boot1 version tag */
+#define VK_BAR1_UCODE_VER_TAG 0x6170
+#define VK_BAR1_BOOT1_VER_TAG 0x61b0
+#define VK_BAR1_VER_TAG_SIZE 64
+
+/* Memory to hold the DMA buffer memory address allocated for boot2 download */
+#define VK_BAR1_DMA_BUF_OFF_HI 0x61e0
+#define VK_BAR1_DMA_BUF_OFF_LO (VK_BAR1_DMA_BUF_OFF_HI + 4)
+#define VK_BAR1_DMA_BUF_SZ (VK_BAR1_DMA_BUF_OFF_HI + 8)
+
+/* Scratch memory allocated on host for VK */
+#define VK_BAR1_SCRATCH_OFF_HI 0x61f0
+#define VK_BAR1_SCRATCH_OFF_LO (VK_BAR1_SCRATCH_OFF_HI + 4)
+#define VK_BAR1_SCRATCH_SZ_ADDR (VK_BAR1_SCRATCH_OFF_HI + 8)
+#define VK_BAR1_SCRATCH_DEF_NR_PAGES 32
+
+/* BAR1 DAUTH info */
+#define VK_BAR1_DAUTH_BASE_ADDR 0x6200
+#define VK_BAR1_DAUTH_STORE_SIZE 0x48
+#define VK_BAR1_DAUTH_VALID_SIZE 0x8
+#define VK_BAR1_DAUTH_MAX 4
+#define VK_BAR1_DAUTH_STORE_ADDR(x) \
+ (VK_BAR1_DAUTH_BASE_ADDR + \
+ (x) * (VK_BAR1_DAUTH_STORE_SIZE + VK_BAR1_DAUTH_VALID_SIZE))
+#define VK_BAR1_DAUTH_VALID_ADDR(x) \
+ (VK_BAR1_DAUTH_STORE_ADDR(x) + VK_BAR1_DAUTH_STORE_SIZE)
+
+/* BAR1 SOTP AUTH and REVID info */
+#define VK_BAR1_SOTP_REVID_BASE_ADDR 0x6340
+#define VK_BAR1_SOTP_REVID_SIZE 0x10
+#define VK_BAR1_SOTP_REVID_MAX 2
+#define VK_BAR1_SOTP_REVID_ADDR(x) \
+ (VK_BAR1_SOTP_REVID_BASE_ADDR + (x) * VK_BAR1_SOTP_REVID_SIZE)
+
+/* VK device supports a maximum of 3 bars */
+#define MAX_BAR 3
+
+/* default number of msg blk for inband SGL */
+#define BCM_VK_DEF_IB_SGL_BLK_LEN 16
+#define BCM_VK_IB_SGL_BLK_MAX 24
+
+enum pci_barno {
+ BAR_0 = 0,
+ BAR_1,
+ BAR_2
+};
+
+#ifdef CONFIG_BCM_VK_TTY
+#define BCM_VK_NUM_TTY 2
+#else
+#define BCM_VK_NUM_TTY 0
+#endif
+
+struct bcm_vk_tty {
+ struct tty_port port;
+ u32 to_offset; /* bar offset to use */
+ u32 to_size; /* to VK buffer size */
+ u32 wr; /* write offset shadow */
+ u32 from_offset; /* bar offset to use */
+ u32 from_size; /* from VK buffer size */
+ u32 rd; /* read offset shadow */
+ pid_t pid;
+ bool irq_enabled;
+ bool is_opened; /* tracks tty open/close */
+};
+
+/* VK device max power state, supports 3, full, reduced and low */
+#define MAX_OPP 3
+#define MAX_CARD_INFO_TAG_SIZE 64
+
+struct bcm_vk_card_info {
+ u32 version;
+ char os_tag[MAX_CARD_INFO_TAG_SIZE];
+ char cmpt_tag[MAX_CARD_INFO_TAG_SIZE];
+ u32 cpu_freq_mhz;
+ u32 cpu_scale[MAX_OPP];
+ u32 ddr_freq_mhz;
+ u32 ddr_size_MB;
+ u32 video_core_freq_mhz;
+};
+
+/* DAUTH related info */
+struct bcm_vk_dauth_key {
+ char store[VK_BAR1_DAUTH_STORE_SIZE];
+ char valid[VK_BAR1_DAUTH_VALID_SIZE];
+};
+
+struct bcm_vk_dauth_info {
+ struct bcm_vk_dauth_key keys[VK_BAR1_DAUTH_MAX];
+};
+
+/*
+ * Control structure of logging messages from the card. This
+ * buffer is for logmsg that comes from vk
+ */
+struct bcm_vk_peer_log {
+ u32 rd_idx;
+ u32 wr_idx;
+ u32 buf_size;
+ u32 mask;
+ char data[];
+};
+
+/* max buf size allowed */
+#define BCM_VK_PEER_LOG_BUF_MAX SZ_16K
+/* max size per line of peer log */
+#define BCM_VK_PEER_LOG_LINE_MAX 256
+
+/*
+ * single entry for processing type + utilization
+ */
+#define BCM_VK_PROC_TYPE_TAG_LEN 8
+struct bcm_vk_proc_mon_entry_t {
+ char tag[BCM_VK_PROC_TYPE_TAG_LEN];
+ u32 used;
+ u32 max; /**< max capacity */
+};
+
+/**
+ * Structure for run time utilization
+ */
+#define BCM_VK_PROC_MON_MAX 8 /* max entries supported */
+struct bcm_vk_proc_mon_info {
+ u32 num; /**< no of entries */
+ u32 entry_size; /**< per entry size */
+ struct bcm_vk_proc_mon_entry_t entries[BCM_VK_PROC_MON_MAX];
+};
+
+struct bcm_vk_hb_ctrl {
+ struct timer_list timer;
+ u32 last_uptime;
+ u32 lost_cnt;
+};
+
+struct bcm_vk_alert {
+ u16 flags;
+ u16 notfs;
+};
+
+/* some alert counters that the driver will keep track */
+struct bcm_vk_alert_cnts {
+ u16 ecc;
+ u16 uecc;
+};
+
+struct bcm_vk {
+ struct pci_dev *pdev;
+ void __iomem *bar[MAX_BAR];
+ int num_irqs;
+
+ struct bcm_vk_card_info card_info;
+ struct bcm_vk_proc_mon_info proc_mon_info;
+ struct bcm_vk_dauth_info dauth_info;
+
+ /* mutex to protect the ioctls */
+ struct mutex mutex;
+ struct miscdevice miscdev;
+ int devid; /* dev id allocated */
+
+#ifdef CONFIG_BCM_VK_TTY
+ struct tty_driver *tty_drv;
+ struct timer_list serial_timer;
+ struct bcm_vk_tty tty[BCM_VK_NUM_TTY];
+ struct workqueue_struct *tty_wq_thread;
+ struct work_struct tty_wq_work;
+#endif
+
+ /* Reference-counting to handle file operations */
+ struct kref kref;
+
+ spinlock_t msg_id_lock; /* Spinlock for msg_id */
+ u16 msg_id;
+ DECLARE_BITMAP(bmap, VK_MSG_ID_BITMAP_SIZE);
+ spinlock_t ctx_lock; /* Spinlock for component context */
+ struct bcm_vk_ctx ctx[VK_CMPT_CTX_MAX];
+ struct bcm_vk_ht_entry pid_ht[VK_PID_HT_SZ];
+ pid_t reset_pid; /* process that issue reset */
+
+ atomic_t msgq_inited; /* indicate if info has been synced with vk */
+ struct bcm_vk_msg_chan to_v_msg_chan;
+ struct bcm_vk_msg_chan to_h_msg_chan;
+
+ struct workqueue_struct *wq_thread;
+ struct work_struct wq_work; /* work queue for deferred job */
+ unsigned long wq_offload[1]; /* various flags on wq requested */
+ void *tdma_vaddr; /* test dma segment virtual addr */
+ dma_addr_t tdma_addr; /* test dma segment bus addr */
+
+ struct notifier_block panic_nb;
+ u32 ib_sgl_size; /* size allocated for inband sgl insertion */
+
+ /* heart beat mechanism control structure */
+ struct bcm_vk_hb_ctrl hb_ctrl;
+ /* house-keeping variable of error logs */
+ spinlock_t host_alert_lock; /* protection to access host_alert struct */
+ struct bcm_vk_alert host_alert;
+ struct bcm_vk_alert peer_alert; /* bits set by the card */
+ struct bcm_vk_alert_cnts alert_cnts;
+
+ /* offset of the peer log control in BAR2 */
+ u32 peerlog_off;
+ struct bcm_vk_peer_log peerlog_info; /* record of peer log info */
+ /* offset of processing monitoring info in BAR2 */
+ u32 proc_mon_off;
+};
+
+/* wq offload work items bits definitions */
+enum bcm_vk_wq_offload_flags {
+ BCM_VK_WQ_DWNLD_PEND = 0,
+ BCM_VK_WQ_DWNLD_AUTO = 1,
+ BCM_VK_WQ_NOTF_PEND = 2,
+};
+
+/* a macro to get an individual field with mask and shift */
+#define BCM_VK_EXTRACT_FIELD(_field, _reg, _mask, _shift) \
+ (_field = (((_reg) >> (_shift)) & (_mask)))
+
+struct bcm_vk_entry {
+ const u32 mask;
+ const u32 exp_val;
+ const char *str;
+};
+
+/* alerts that could be generated from peer */
+#define BCM_VK_PEER_ERR_NUM 12
+extern struct bcm_vk_entry const bcm_vk_peer_err[BCM_VK_PEER_ERR_NUM];
+/* alerts detected by the host */
+#define BCM_VK_HOST_ERR_NUM 3
+extern struct bcm_vk_entry const bcm_vk_host_err[BCM_VK_HOST_ERR_NUM];
+
+/*
+ * check if PCIe interface is down on read. Use it when it is
+ * certain that _val should never be all ones.
+ */
+#define BCM_VK_INTF_IS_DOWN(val) ((val) == 0xffffffff)
+
+static inline u32 vkread32(struct bcm_vk *vk, enum pci_barno bar, u64 offset)
+{
+ return readl(vk->bar[bar] + offset);
+}
+
+static inline void vkwrite32(struct bcm_vk *vk,
+ u32 value,
+ enum pci_barno bar,
+ u64 offset)
+{
+ writel(value, vk->bar[bar] + offset);
+}
+
+static inline u8 vkread8(struct bcm_vk *vk, enum pci_barno bar, u64 offset)
+{
+ return readb(vk->bar[bar] + offset);
+}
+
+static inline void vkwrite8(struct bcm_vk *vk,
+ u8 value,
+ enum pci_barno bar,
+ u64 offset)
+{
+ writeb(value, vk->bar[bar] + offset);
+}
+
+static inline bool bcm_vk_msgq_marker_valid(struct bcm_vk *vk)
+{
+ u32 rdy_marker = 0;
+ u32 fw_status;
+
+ fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS);
+
+ if ((fw_status & VK_FWSTS_READY) == VK_FWSTS_READY)
+ rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY);
+
+ return (rdy_marker == VK_BAR1_MSGQ_RDY_MARKER);
+}
+
+int bcm_vk_open(struct inode *inode, struct file *p_file);
+ssize_t bcm_vk_read(struct file *p_file, char __user *buf, size_t count,
+ loff_t *f_pos);
+ssize_t bcm_vk_write(struct file *p_file, const char __user *buf,
+ size_t count, loff_t *f_pos);
+__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait);
+int bcm_vk_release(struct inode *inode, struct file *p_file);
+void bcm_vk_release_data(struct kref *kref);
+irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id);
+irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id);
+irqreturn_t bcm_vk_tty_irqhandler(int irq, void *dev_id);
+int bcm_vk_msg_init(struct bcm_vk *vk);
+void bcm_vk_msg_remove(struct bcm_vk *vk);
+void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk);
+int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync);
+void bcm_vk_blk_drv_access(struct bcm_vk *vk);
+s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk);
+int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type,
+ const pid_t pid, const u32 q_num);
+void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val);
+int bcm_vk_auto_load_all_images(struct bcm_vk *vk);
+void bcm_vk_hb_init(struct bcm_vk *vk);
+void bcm_vk_hb_deinit(struct bcm_vk *vk);
+void bcm_vk_handle_notf(struct bcm_vk *vk);
+bool bcm_vk_drv_access_ok(struct bcm_vk *vk);
+void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask);
+
+#ifdef CONFIG_BCM_VK_TTY
+int bcm_vk_tty_init(struct bcm_vk *vk, char *name);
+void bcm_vk_tty_exit(struct bcm_vk *vk);
+void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk);
+void bcm_vk_tty_wq_exit(struct bcm_vk *vk);
+
+static inline void bcm_vk_tty_set_irq_enabled(struct bcm_vk *vk, int index)
+{
+ vk->tty[index].irq_enabled = true;
+}
+#else
+static inline int bcm_vk_tty_init(struct bcm_vk *vk, char *name)
+{
+ return 0;
+}
+
+static inline void bcm_vk_tty_exit(struct bcm_vk *vk)
+{
+}
+
+static inline void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk)
+{
+}
+
+static inline void bcm_vk_tty_wq_exit(struct bcm_vk *vk)
+{
+}
+
+static inline void bcm_vk_tty_set_irq_enabled(struct bcm_vk *vk, int index)
+{
+}
+#endif /* CONFIG_BCM_VK_TTY */
+
+#endif
diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c
new file mode 100644
index 000000000..d4a961377
--- /dev/null
+++ b/drivers/misc/bcm-vk/bcm_vk_dev.c
@@ -0,0 +1,1652 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/firmware.h>
+#include <linux/fs.h>
+#include <linux/idr.h>
+#include <linux/interrupt.h>
+#include <linux/panic_notifier.h>
+#include <linux/kref.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <uapi/linux/misc/bcm_vk.h>
+
+#include "bcm_vk.h"
+
+#define PCI_DEVICE_ID_VALKYRIE 0x5e87
+#define PCI_DEVICE_ID_VIPER 0x5e88
+
+static DEFINE_IDA(bcm_vk_ida);
+
+enum soc_idx {
+ VALKYRIE_A0 = 0,
+ VALKYRIE_B0,
+ VIPER,
+ VK_IDX_INVALID
+};
+
+enum img_idx {
+ IMG_PRI = 0,
+ IMG_SEC,
+ IMG_PER_TYPE_MAX
+};
+
+struct load_image_entry {
+ const u32 image_type;
+ const char *image_name[IMG_PER_TYPE_MAX];
+};
+
+#define NUM_BOOT_STAGES 2
+/* default firmware images names */
+static const struct load_image_entry image_tab[][NUM_BOOT_STAGES] = {
+ [VALKYRIE_A0] = {
+ {VK_IMAGE_TYPE_BOOT1, {"vk_a0-boot1.bin", "vk-boot1.bin"}},
+ {VK_IMAGE_TYPE_BOOT2, {"vk_a0-boot2.bin", "vk-boot2.bin"}}
+ },
+ [VALKYRIE_B0] = {
+ {VK_IMAGE_TYPE_BOOT1, {"vk_b0-boot1.bin", "vk-boot1.bin"}},
+ {VK_IMAGE_TYPE_BOOT2, {"vk_b0-boot2.bin", "vk-boot2.bin"}}
+ },
+
+ [VIPER] = {
+ {VK_IMAGE_TYPE_BOOT1, {"vp-boot1.bin", ""}},
+ {VK_IMAGE_TYPE_BOOT2, {"vp-boot2.bin", ""}}
+ },
+};
+
+/* Location of memory base addresses of interest in BAR1 */
+/* Load Boot1 to start of ITCM */
+#define BAR1_CODEPUSH_BASE_BOOT1 0x100000
+
+/* Allow minimum 1s for Load Image timeout responses */
+#define LOAD_IMAGE_TIMEOUT_MS (1 * MSEC_PER_SEC)
+
+/* Image startup timeouts */
+#define BOOT1_STARTUP_TIMEOUT_MS (5 * MSEC_PER_SEC)
+#define BOOT2_STARTUP_TIMEOUT_MS (10 * MSEC_PER_SEC)
+
+/* 1ms wait for checking the transfer complete status */
+#define TXFR_COMPLETE_TIMEOUT_MS 1
+
+/* MSIX usages */
+#define VK_MSIX_MSGQ_MAX 3
+#define VK_MSIX_NOTF_MAX 1
+#define VK_MSIX_TTY_MAX BCM_VK_NUM_TTY
+#define VK_MSIX_IRQ_MAX (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX + \
+ VK_MSIX_TTY_MAX)
+#define VK_MSIX_IRQ_MIN_REQ (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX)
+
+/* Number of bits set in DMA mask*/
+#define BCM_VK_DMA_BITS 64
+
+/* Ucode boot wait time */
+#define BCM_VK_UCODE_BOOT_US (100 * USEC_PER_MSEC)
+/* 50% margin */
+#define BCM_VK_UCODE_BOOT_MAX_US ((BCM_VK_UCODE_BOOT_US * 3) >> 1)
+
+/* deinit time for the card os after receiving doorbell */
+#define BCM_VK_DEINIT_TIME_MS (2 * MSEC_PER_SEC)
+
+/*
+ * module parameters
+ */
+static bool auto_load = true;
+module_param(auto_load, bool, 0444);
+MODULE_PARM_DESC(auto_load,
+ "Load images automatically at PCIe probe time.\n");
+static uint nr_scratch_pages = VK_BAR1_SCRATCH_DEF_NR_PAGES;
+module_param(nr_scratch_pages, uint, 0444);
+MODULE_PARM_DESC(nr_scratch_pages,
+ "Number of pre allocated DMAable coherent pages.\n");
+static uint nr_ib_sgl_blk = BCM_VK_DEF_IB_SGL_BLK_LEN;
+module_param(nr_ib_sgl_blk, uint, 0444);
+MODULE_PARM_DESC(nr_ib_sgl_blk,
+ "Number of in-band msg blks for short SGL.\n");
+
+/*
+ * alerts that could be generated from peer
+ */
+const struct bcm_vk_entry bcm_vk_peer_err[BCM_VK_PEER_ERR_NUM] = {
+ {ERR_LOG_UECC, ERR_LOG_UECC, "uecc"},
+ {ERR_LOG_SSIM_BUSY, ERR_LOG_SSIM_BUSY, "ssim_busy"},
+ {ERR_LOG_AFBC_BUSY, ERR_LOG_AFBC_BUSY, "afbc_busy"},
+ {ERR_LOG_HIGH_TEMP_ERR, ERR_LOG_HIGH_TEMP_ERR, "high_temp"},
+ {ERR_LOG_WDOG_TIMEOUT, ERR_LOG_WDOG_TIMEOUT, "wdog_timeout"},
+ {ERR_LOG_SYS_FAULT, ERR_LOG_SYS_FAULT, "sys_fault"},
+ {ERR_LOG_RAMDUMP, ERR_LOG_RAMDUMP, "ramdump"},
+ {ERR_LOG_COP_WDOG_TIMEOUT, ERR_LOG_COP_WDOG_TIMEOUT,
+ "cop_wdog_timeout"},
+ {ERR_LOG_MEM_ALLOC_FAIL, ERR_LOG_MEM_ALLOC_FAIL, "malloc_fail warn"},
+ {ERR_LOG_LOW_TEMP_WARN, ERR_LOG_LOW_TEMP_WARN, "low_temp warn"},
+ {ERR_LOG_ECC, ERR_LOG_ECC, "ecc"},
+ {ERR_LOG_IPC_DWN, ERR_LOG_IPC_DWN, "ipc_down"},
+};
+
+/* alerts detected by the host */
+const struct bcm_vk_entry bcm_vk_host_err[BCM_VK_HOST_ERR_NUM] = {
+ {ERR_LOG_HOST_PCIE_DWN, ERR_LOG_HOST_PCIE_DWN, "PCIe_down"},
+ {ERR_LOG_HOST_HB_FAIL, ERR_LOG_HOST_HB_FAIL, "hb_fail"},
+ {ERR_LOG_HOST_INTF_V_FAIL, ERR_LOG_HOST_INTF_V_FAIL, "intf_ver_fail"},
+};
+
+irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id)
+{
+ struct bcm_vk *vk = dev_id;
+
+ if (!bcm_vk_drv_access_ok(vk)) {
+ dev_err(&vk->pdev->dev,
+ "Interrupt %d received when msgq not inited\n", irq);
+ goto skip_schedule_work;
+ }
+
+ /* if notification is not pending, set bit and schedule work */
+ if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0)
+ queue_work(vk->wq_thread, &vk->wq_work);
+
+skip_schedule_work:
+ return IRQ_HANDLED;
+}
+
+static int bcm_vk_intf_ver_chk(struct bcm_vk *vk)
+{
+ struct device *dev = &vk->pdev->dev;
+ u32 reg;
+ u16 major, minor;
+ int ret = 0;
+
+ /* read interface register */
+ reg = vkread32(vk, BAR_0, BAR_INTF_VER);
+ major = (reg >> BAR_INTF_VER_MAJOR_SHIFT) & BAR_INTF_VER_MASK;
+ minor = reg & BAR_INTF_VER_MASK;
+
+ /*
+ * if major number is 0, it is pre-release and it would be allowed
+ * to continue, else, check versions accordingly
+ */
+ if (!major) {
+ dev_warn(dev, "Pre-release major.minor=%d.%d - drv %d.%d\n",
+ major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR);
+ } else if (major != SEMANTIC_MAJOR) {
+ dev_err(dev,
+ "Intf major.minor=%d.%d rejected - drv %d.%d\n",
+ major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR);
+ bcm_vk_set_host_alert(vk, ERR_LOG_HOST_INTF_V_FAIL);
+ ret = -EPFNOSUPPORT;
+ } else {
+ dev_dbg(dev,
+ "Intf major.minor=%d.%d passed - drv %d.%d\n",
+ major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR);
+ }
+ return ret;
+}
+
+static void bcm_vk_log_notf(struct bcm_vk *vk,
+ struct bcm_vk_alert *alert,
+ struct bcm_vk_entry const *entry_tab,
+ const u32 table_size)
+{
+ u32 i;
+ u32 masked_val, latched_val;
+ struct bcm_vk_entry const *entry;
+ u32 reg;
+ u16 ecc_mem_err, uecc_mem_err;
+ struct device *dev = &vk->pdev->dev;
+
+ for (i = 0; i < table_size; i++) {
+ entry = &entry_tab[i];
+ masked_val = entry->mask & alert->notfs;
+ latched_val = entry->mask & alert->flags;
+
+ if (masked_val == ERR_LOG_UECC) {
+ /*
+ * if there is difference between stored cnt and it
+ * is greater than threshold, log it.
+ */
+ reg = vkread32(vk, BAR_0, BAR_CARD_ERR_MEM);
+ BCM_VK_EXTRACT_FIELD(uecc_mem_err, reg,
+ BCM_VK_MEM_ERR_FIELD_MASK,
+ BCM_VK_UECC_MEM_ERR_SHIFT);
+ if ((uecc_mem_err != vk->alert_cnts.uecc) &&
+ (uecc_mem_err >= BCM_VK_UECC_THRESHOLD))
+ dev_info(dev,
+ "ALERT! %s.%d uecc RAISED - ErrCnt %d\n",
+ DRV_MODULE_NAME, vk->devid,
+ uecc_mem_err);
+ vk->alert_cnts.uecc = uecc_mem_err;
+ } else if (masked_val == ERR_LOG_ECC) {
+ reg = vkread32(vk, BAR_0, BAR_CARD_ERR_MEM);
+ BCM_VK_EXTRACT_FIELD(ecc_mem_err, reg,
+ BCM_VK_MEM_ERR_FIELD_MASK,
+ BCM_VK_ECC_MEM_ERR_SHIFT);
+ if ((ecc_mem_err != vk->alert_cnts.ecc) &&
+ (ecc_mem_err >= BCM_VK_ECC_THRESHOLD))
+ dev_info(dev, "ALERT! %s.%d ecc RAISED - ErrCnt %d\n",
+ DRV_MODULE_NAME, vk->devid,
+ ecc_mem_err);
+ vk->alert_cnts.ecc = ecc_mem_err;
+ } else if (masked_val != latched_val) {
+ /* print a log as info */
+ dev_info(dev, "ALERT! %s.%d %s %s\n",
+ DRV_MODULE_NAME, vk->devid, entry->str,
+ masked_val ? "RAISED" : "CLEARED");
+ }
+ }
+}
+
+static void bcm_vk_dump_peer_log(struct bcm_vk *vk)
+{
+ struct bcm_vk_peer_log log;
+ struct bcm_vk_peer_log *log_info = &vk->peerlog_info;
+ char loc_buf[BCM_VK_PEER_LOG_LINE_MAX];
+ int cnt;
+ struct device *dev = &vk->pdev->dev;
+ unsigned int data_offset;
+
+ memcpy_fromio(&log, vk->bar[BAR_2] + vk->peerlog_off, sizeof(log));
+
+ dev_dbg(dev, "Peer PANIC: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n",
+ log.buf_size, log.mask, log.rd_idx, log.wr_idx);
+
+ if (!log_info->buf_size) {
+ dev_err(dev, "Peer log dump disabled - skipped!\n");
+ return;
+ }
+
+ /* perform range checking for rd/wr idx */
+ if ((log.rd_idx > log_info->mask) ||
+ (log.wr_idx > log_info->mask) ||
+ (log.buf_size != log_info->buf_size) ||
+ (log.mask != log_info->mask)) {
+ dev_err(dev,
+ "Corrupted Ptrs: Size 0x%x(0x%x) Mask 0x%x(0x%x) [Rd Wr] = [%d %d], skip log dump.\n",
+ log_info->buf_size, log.buf_size,
+ log_info->mask, log.mask,
+ log.rd_idx, log.wr_idx);
+ return;
+ }
+
+ cnt = 0;
+ data_offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log);
+ loc_buf[BCM_VK_PEER_LOG_LINE_MAX - 1] = '\0';
+ while (log.rd_idx != log.wr_idx) {
+ loc_buf[cnt] = vkread8(vk, BAR_2, data_offset + log.rd_idx);
+
+ if ((loc_buf[cnt] == '\0') ||
+ (cnt == (BCM_VK_PEER_LOG_LINE_MAX - 1))) {
+ dev_err(dev, "%s", loc_buf);
+ cnt = 0;
+ } else {
+ cnt++;
+ }
+ log.rd_idx = (log.rd_idx + 1) & log.mask;
+ }
+ /* update rd idx at the end */
+ vkwrite32(vk, log.rd_idx, BAR_2,
+ vk->peerlog_off + offsetof(struct bcm_vk_peer_log, rd_idx));
+}
+
+void bcm_vk_handle_notf(struct bcm_vk *vk)
+{
+ u32 reg;
+ struct bcm_vk_alert alert;
+ bool intf_down;
+ unsigned long flags;
+
+ /* handle peer alerts and then locally detected ones */
+ reg = vkread32(vk, BAR_0, BAR_CARD_ERR_LOG);
+ intf_down = BCM_VK_INTF_IS_DOWN(reg);
+ if (!intf_down) {
+ vk->peer_alert.notfs = reg;
+ bcm_vk_log_notf(vk, &vk->peer_alert, bcm_vk_peer_err,
+ ARRAY_SIZE(bcm_vk_peer_err));
+ vk->peer_alert.flags = vk->peer_alert.notfs;
+ } else {
+ /* turn off access */
+ bcm_vk_blk_drv_access(vk);
+ }
+
+ /* check and make copy of alert with lock and then free lock */
+ spin_lock_irqsave(&vk->host_alert_lock, flags);
+ if (intf_down)
+ vk->host_alert.notfs |= ERR_LOG_HOST_PCIE_DWN;
+
+ alert = vk->host_alert;
+ vk->host_alert.flags = vk->host_alert.notfs;
+ spin_unlock_irqrestore(&vk->host_alert_lock, flags);
+
+ /* call display with copy */
+ bcm_vk_log_notf(vk, &alert, bcm_vk_host_err,
+ ARRAY_SIZE(bcm_vk_host_err));
+
+ /*
+ * If it is a sys fault or heartbeat timeout, we would like extract
+ * log msg from the card so that we would know what is the last fault
+ */
+ if (!intf_down &&
+ ((vk->host_alert.flags & ERR_LOG_HOST_HB_FAIL) ||
+ (vk->peer_alert.flags & ERR_LOG_SYS_FAULT)))
+ bcm_vk_dump_peer_log(vk);
+}
+
+static inline int bcm_vk_wait(struct bcm_vk *vk, enum pci_barno bar,
+ u64 offset, u32 mask, u32 value,
+ unsigned long timeout_ms)
+{
+ struct device *dev = &vk->pdev->dev;
+ unsigned long start_time;
+ unsigned long timeout;
+ u32 rd_val, boot_status;
+
+ start_time = jiffies;
+ timeout = start_time + msecs_to_jiffies(timeout_ms);
+
+ do {
+ rd_val = vkread32(vk, bar, offset);
+ dev_dbg(dev, "BAR%d Offset=0x%llx: 0x%x\n",
+ bar, offset, rd_val);
+
+ /* check for any boot err condition */
+ boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+ if (boot_status & BOOT_ERR_MASK) {
+ dev_err(dev, "Boot Err 0x%x, progress 0x%x after %d ms\n",
+ (boot_status & BOOT_ERR_MASK) >> BOOT_ERR_SHIFT,
+ boot_status & BOOT_PROG_MASK,
+ jiffies_to_msecs(jiffies - start_time));
+ return -EFAULT;
+ }
+
+ if (time_after(jiffies, timeout))
+ return -ETIMEDOUT;
+
+ cpu_relax();
+ cond_resched();
+ } while ((rd_val & mask) != value);
+
+ return 0;
+}
+
+static void bcm_vk_get_card_info(struct bcm_vk *vk)
+{
+ struct device *dev = &vk->pdev->dev;
+ u32 offset;
+ int i;
+ u8 *dst;
+ struct bcm_vk_card_info *info = &vk->card_info;
+
+ /* first read the offset from spare register */
+ offset = vkread32(vk, BAR_0, BAR_CARD_STATIC_INFO);
+ offset &= (pci_resource_len(vk->pdev, BAR_2 * 2) - 1);
+
+ /* based on the offset, read info to internal card info structure */
+ dst = (u8 *)info;
+ for (i = 0; i < sizeof(*info); i++)
+ *dst++ = vkread8(vk, BAR_2, offset++);
+
+#define CARD_INFO_LOG_FMT "version : %x\n" \
+ "os_tag : %s\n" \
+ "cmpt_tag : %s\n" \
+ "cpu_freq : %d MHz\n" \
+ "cpu_scale : %d full, %d lowest\n" \
+ "ddr_freq : %d MHz\n" \
+ "ddr_size : %d MB\n" \
+ "video_freq: %d MHz\n"
+ dev_dbg(dev, CARD_INFO_LOG_FMT, info->version, info->os_tag,
+ info->cmpt_tag, info->cpu_freq_mhz, info->cpu_scale[0],
+ info->cpu_scale[MAX_OPP - 1], info->ddr_freq_mhz,
+ info->ddr_size_MB, info->video_core_freq_mhz);
+
+ /*
+ * get the peer log pointer, only need the offset, and get record
+ * of the log buffer information which would be used for checking
+ * before dump, in case the BAR2 memory has been corrupted.
+ */
+ vk->peerlog_off = offset;
+ memcpy_fromio(&vk->peerlog_info, vk->bar[BAR_2] + vk->peerlog_off,
+ sizeof(vk->peerlog_info));
+
+ /*
+ * Do a range checking and if out of bound, the record will be zeroed
+ * which guarantees that nothing would be dumped. In other words,
+ * peer dump is disabled.
+ */
+ if ((vk->peerlog_info.buf_size > BCM_VK_PEER_LOG_BUF_MAX) ||
+ (vk->peerlog_info.mask != (vk->peerlog_info.buf_size - 1)) ||
+ (vk->peerlog_info.rd_idx > vk->peerlog_info.mask) ||
+ (vk->peerlog_info.wr_idx > vk->peerlog_info.mask)) {
+ dev_err(dev, "Peer log disabled - range error: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n",
+ vk->peerlog_info.buf_size,
+ vk->peerlog_info.mask,
+ vk->peerlog_info.rd_idx,
+ vk->peerlog_info.wr_idx);
+ memset(&vk->peerlog_info, 0, sizeof(vk->peerlog_info));
+ } else {
+ dev_dbg(dev, "Peer log: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n",
+ vk->peerlog_info.buf_size,
+ vk->peerlog_info.mask,
+ vk->peerlog_info.rd_idx,
+ vk->peerlog_info.wr_idx);
+ }
+}
+
+static void bcm_vk_get_proc_mon_info(struct bcm_vk *vk)
+{
+ struct device *dev = &vk->pdev->dev;
+ struct bcm_vk_proc_mon_info *mon = &vk->proc_mon_info;
+ u32 num, entry_size, offset, buf_size;
+ u8 *dst;
+
+ /* calculate offset which is based on peerlog offset */
+ buf_size = vkread32(vk, BAR_2,
+ vk->peerlog_off
+ + offsetof(struct bcm_vk_peer_log, buf_size));
+ offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log)
+ + buf_size;
+
+ /* first read the num and entry size */
+ num = vkread32(vk, BAR_2, offset);
+ entry_size = vkread32(vk, BAR_2, offset + sizeof(num));
+
+ /* check for max allowed */
+ if (num > BCM_VK_PROC_MON_MAX) {
+ dev_err(dev, "Processing monitoring entry %d exceeds max %d\n",
+ num, BCM_VK_PROC_MON_MAX);
+ return;
+ }
+ mon->num = num;
+ mon->entry_size = entry_size;
+
+ vk->proc_mon_off = offset;
+
+ /* read it once that will capture those static info */
+ dst = (u8 *)&mon->entries[0];
+ offset += sizeof(num) + sizeof(entry_size);
+ memcpy_fromio(dst, vk->bar[BAR_2] + offset, num * entry_size);
+}
+
+static int bcm_vk_sync_card_info(struct bcm_vk *vk)
+{
+ u32 rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY);
+
+ /* check for marker, but allow diags mode to skip sync */
+ if (!bcm_vk_msgq_marker_valid(vk))
+ return (rdy_marker == VK_BAR1_DIAG_RDY_MARKER ? 0 : -EINVAL);
+
+ /*
+ * Write down scratch addr which is used for DMA. For
+ * signed part, BAR1 is accessible only after boot2 has come
+ * up
+ */
+ if (vk->tdma_addr) {
+ vkwrite32(vk, (u64)vk->tdma_addr >> 32, BAR_1,
+ VK_BAR1_SCRATCH_OFF_HI);
+ vkwrite32(vk, (u32)vk->tdma_addr, BAR_1,
+ VK_BAR1_SCRATCH_OFF_LO);
+ vkwrite32(vk, nr_scratch_pages * PAGE_SIZE, BAR_1,
+ VK_BAR1_SCRATCH_SZ_ADDR);
+ }
+
+ /* get static card info, only need to read once */
+ bcm_vk_get_card_info(vk);
+
+ /* get the proc mon info once */
+ bcm_vk_get_proc_mon_info(vk);
+
+ return 0;
+}
+
+void bcm_vk_blk_drv_access(struct bcm_vk *vk)
+{
+ int i;
+
+ /*
+ * kill all the apps except for the process that is resetting.
+ * If not called during reset, reset_pid will be 0, and all will be
+ * killed.
+ */
+ spin_lock(&vk->ctx_lock);
+
+ /* set msgq_inited to 0 so that all rd/wr will be blocked */
+ atomic_set(&vk->msgq_inited, 0);
+
+ for (i = 0; i < VK_PID_HT_SZ; i++) {
+ struct bcm_vk_ctx *ctx;
+
+ list_for_each_entry(ctx, &vk->pid_ht[i].head, node) {
+ if (ctx->pid != vk->reset_pid) {
+ dev_dbg(&vk->pdev->dev,
+ "Send kill signal to pid %d\n",
+ ctx->pid);
+ kill_pid(find_vpid(ctx->pid), SIGKILL, 1);
+ }
+ }
+ }
+ bcm_vk_tty_terminate_tty_user(vk);
+ spin_unlock(&vk->ctx_lock);
+}
+
+static void bcm_vk_buf_notify(struct bcm_vk *vk, void *bufp,
+ dma_addr_t host_buf_addr, u32 buf_size)
+{
+ /* update the dma address to the card */
+ vkwrite32(vk, (u64)host_buf_addr >> 32, BAR_1,
+ VK_BAR1_DMA_BUF_OFF_HI);
+ vkwrite32(vk, (u32)host_buf_addr, BAR_1,
+ VK_BAR1_DMA_BUF_OFF_LO);
+ vkwrite32(vk, buf_size, BAR_1, VK_BAR1_DMA_BUF_SZ);
+}
+
+static int bcm_vk_load_image_by_type(struct bcm_vk *vk, u32 load_type,
+ const char *filename)
+{
+ struct device *dev = &vk->pdev->dev;
+ const struct firmware *fw = NULL;
+ void *bufp = NULL;
+ size_t max_buf, offset;
+ int ret;
+ u64 offset_codepush;
+ u32 codepush;
+ u32 value;
+ dma_addr_t boot_dma_addr;
+ bool is_stdalone;
+
+ if (load_type == VK_IMAGE_TYPE_BOOT1) {
+ /*
+ * After POR, enable VK soft BOOTSRC so bootrom do not clear
+ * the pushed image (the TCM memories).
+ */
+ value = vkread32(vk, BAR_0, BAR_BOOTSRC_SELECT);
+ value |= BOOTSRC_SOFT_ENABLE;
+ vkwrite32(vk, value, BAR_0, BAR_BOOTSRC_SELECT);
+
+ codepush = CODEPUSH_BOOTSTART + CODEPUSH_BOOT1_ENTRY;
+ offset_codepush = BAR_CODEPUSH_SBL;
+
+ /* Write a 1 to request SRAM open bit */
+ vkwrite32(vk, CODEPUSH_BOOTSTART, BAR_0, offset_codepush);
+
+ /* Wait for VK to respond */
+ ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, SRAM_OPEN,
+ SRAM_OPEN, LOAD_IMAGE_TIMEOUT_MS);
+ if (ret < 0) {
+ dev_err(dev, "boot1 wait SRAM err - ret(%d)\n", ret);
+ goto err_buf_out;
+ }
+
+ max_buf = SZ_256K;
+ bufp = dma_alloc_coherent(dev,
+ max_buf,
+ &boot_dma_addr, GFP_KERNEL);
+ if (!bufp) {
+ dev_err(dev, "Error allocating 0x%zx\n", max_buf);
+ ret = -ENOMEM;
+ goto err_buf_out;
+ }
+ } else if (load_type == VK_IMAGE_TYPE_BOOT2) {
+ codepush = CODEPUSH_BOOT2_ENTRY;
+ offset_codepush = BAR_CODEPUSH_SBI;
+
+ /* Wait for VK to respond */
+ ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, DDR_OPEN,
+ DDR_OPEN, LOAD_IMAGE_TIMEOUT_MS);
+ if (ret < 0) {
+ dev_err(dev, "boot2 wait DDR open error - ret(%d)\n",
+ ret);
+ goto err_buf_out;
+ }
+
+ max_buf = SZ_4M;
+ bufp = dma_alloc_coherent(dev,
+ max_buf,
+ &boot_dma_addr, GFP_KERNEL);
+ if (!bufp) {
+ dev_err(dev, "Error allocating 0x%zx\n", max_buf);
+ ret = -ENOMEM;
+ goto err_buf_out;
+ }
+
+ bcm_vk_buf_notify(vk, bufp, boot_dma_addr, max_buf);
+ } else {
+ dev_err(dev, "Error invalid image type 0x%x\n", load_type);
+ ret = -EINVAL;
+ goto err_buf_out;
+ }
+
+ offset = 0;
+ ret = request_partial_firmware_into_buf(&fw, filename, dev,
+ bufp, max_buf, offset);
+ if (ret) {
+ dev_err(dev, "Error %d requesting firmware file: %s\n",
+ ret, filename);
+ goto err_firmware_out;
+ }
+ dev_dbg(dev, "size=0x%zx\n", fw->size);
+ if (load_type == VK_IMAGE_TYPE_BOOT1)
+ memcpy_toio(vk->bar[BAR_1] + BAR1_CODEPUSH_BASE_BOOT1,
+ bufp,
+ fw->size);
+
+ dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", codepush, offset_codepush);
+ vkwrite32(vk, codepush, BAR_0, offset_codepush);
+
+ if (load_type == VK_IMAGE_TYPE_BOOT1) {
+ u32 boot_status;
+
+ /* wait until done */
+ ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS,
+ BOOT1_RUNNING,
+ BOOT1_RUNNING,
+ BOOT1_STARTUP_TIMEOUT_MS);
+
+ boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+ is_stdalone = !BCM_VK_INTF_IS_DOWN(boot_status) &&
+ (boot_status & BOOT_STDALONE_RUNNING);
+ if (ret && !is_stdalone) {
+ dev_err(dev,
+ "Timeout %ld ms waiting for boot1 to come up - ret(%d)\n",
+ BOOT1_STARTUP_TIMEOUT_MS, ret);
+ goto err_firmware_out;
+ } else if (is_stdalone) {
+ u32 reg;
+
+ reg = vkread32(vk, BAR_0, BAR_BOOT1_STDALONE_PROGRESS);
+ if ((reg & BOOT1_STDALONE_PROGRESS_MASK) ==
+ BOOT1_STDALONE_SUCCESS) {
+ dev_info(dev, "Boot1 standalone success\n");
+ ret = 0;
+ } else {
+ dev_err(dev, "Timeout %ld ms - Boot1 standalone failure\n",
+ BOOT1_STARTUP_TIMEOUT_MS);
+ ret = -EINVAL;
+ goto err_firmware_out;
+ }
+ }
+ } else if (load_type == VK_IMAGE_TYPE_BOOT2) {
+ unsigned long timeout;
+
+ timeout = jiffies + msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS);
+
+ /* To send more data to VK than max_buf allowed at a time */
+ do {
+ /*
+ * Check for ack from card. when Ack is received,
+ * it means all the data is received by card.
+ * Exit the loop after ack is received.
+ */
+ ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS,
+ FW_LOADER_ACK_RCVD_ALL_DATA,
+ FW_LOADER_ACK_RCVD_ALL_DATA,
+ TXFR_COMPLETE_TIMEOUT_MS);
+ if (ret == 0) {
+ dev_dbg(dev, "Exit boot2 download\n");
+ break;
+ } else if (ret == -EFAULT) {
+ dev_err(dev, "Error detected during ACK waiting");
+ goto err_firmware_out;
+ }
+
+ /* exit the loop, if there is no response from card */
+ if (time_after(jiffies, timeout)) {
+ dev_err(dev, "Error. No reply from card\n");
+ ret = -ETIMEDOUT;
+ goto err_firmware_out;
+ }
+
+ /* Wait for VK to open BAR space to copy new data */
+ ret = bcm_vk_wait(vk, BAR_0, offset_codepush,
+ codepush, 0,
+ TXFR_COMPLETE_TIMEOUT_MS);
+ if (ret == 0) {
+ offset += max_buf;
+ ret = request_partial_firmware_into_buf
+ (&fw,
+ filename,
+ dev, bufp,
+ max_buf,
+ offset);
+ if (ret) {
+ dev_err(dev,
+ "Error %d requesting firmware file: %s offset: 0x%zx\n",
+ ret, filename, offset);
+ goto err_firmware_out;
+ }
+ dev_dbg(dev, "size=0x%zx\n", fw->size);
+ dev_dbg(dev, "Signaling 0x%x to 0x%llx\n",
+ codepush, offset_codepush);
+ vkwrite32(vk, codepush, BAR_0, offset_codepush);
+ /* reload timeout after every codepush */
+ timeout = jiffies +
+ msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS);
+ } else if (ret == -EFAULT) {
+ dev_err(dev, "Error detected waiting for transfer\n");
+ goto err_firmware_out;
+ }
+ } while (1);
+
+ /* wait for fw status bits to indicate app ready */
+ ret = bcm_vk_wait(vk, BAR_0, VK_BAR_FWSTS,
+ VK_FWSTS_READY,
+ VK_FWSTS_READY,
+ BOOT2_STARTUP_TIMEOUT_MS);
+ if (ret < 0) {
+ dev_err(dev, "Boot2 not ready - ret(%d)\n", ret);
+ goto err_firmware_out;
+ }
+
+ is_stdalone = vkread32(vk, BAR_0, BAR_BOOT_STATUS) &
+ BOOT_STDALONE_RUNNING;
+ if (!is_stdalone) {
+ ret = bcm_vk_intf_ver_chk(vk);
+ if (ret) {
+ dev_err(dev, "failure in intf version check\n");
+ goto err_firmware_out;
+ }
+
+ /*
+ * Next, initialize Message Q if we are loading boot2.
+ * Do a force sync
+ */
+ ret = bcm_vk_sync_msgq(vk, true);
+ if (ret) {
+ dev_err(dev, "Boot2 Error reading comm msg Q info\n");
+ ret = -EIO;
+ goto err_firmware_out;
+ }
+
+ /* sync & channel other info */
+ ret = bcm_vk_sync_card_info(vk);
+ if (ret) {
+ dev_err(dev, "Syncing Card Info failure\n");
+ goto err_firmware_out;
+ }
+ }
+ }
+
+err_firmware_out:
+ release_firmware(fw);
+
+err_buf_out:
+ if (bufp)
+ dma_free_coherent(dev, max_buf, bufp, boot_dma_addr);
+
+ return ret;
+}
+
+static u32 bcm_vk_next_boot_image(struct bcm_vk *vk)
+{
+ u32 boot_status;
+ u32 fw_status;
+ u32 load_type = 0; /* default for unknown */
+
+ boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+ fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS);
+
+ if (!BCM_VK_INTF_IS_DOWN(boot_status) && (boot_status & SRAM_OPEN))
+ load_type = VK_IMAGE_TYPE_BOOT1;
+ else if (boot_status == BOOT1_RUNNING)
+ load_type = VK_IMAGE_TYPE_BOOT2;
+
+ /* Log status so that we know different stages */
+ dev_info(&vk->pdev->dev,
+ "boot-status value for next image: 0x%x : fw-status 0x%x\n",
+ boot_status, fw_status);
+
+ return load_type;
+}
+
+static enum soc_idx get_soc_idx(struct bcm_vk *vk)
+{
+ struct pci_dev *pdev = vk->pdev;
+ enum soc_idx idx = VK_IDX_INVALID;
+ u32 rev;
+ static enum soc_idx const vk_soc_tab[] = { VALKYRIE_A0, VALKYRIE_B0 };
+
+ switch (pdev->device) {
+ case PCI_DEVICE_ID_VALKYRIE:
+ /* get the chip id to decide sub-class */
+ rev = MAJOR_SOC_REV(vkread32(vk, BAR_0, BAR_CHIP_ID));
+ if (rev < ARRAY_SIZE(vk_soc_tab)) {
+ idx = vk_soc_tab[rev];
+ } else {
+ /* Default to A0 firmware for all other chip revs */
+ idx = VALKYRIE_A0;
+ dev_warn(&pdev->dev,
+ "Rev %d not in image lookup table, default to idx=%d\n",
+ rev, idx);
+ }
+ break;
+
+ case PCI_DEVICE_ID_VIPER:
+ idx = VIPER;
+ break;
+
+ default:
+ dev_err(&pdev->dev, "no images for 0x%x\n", pdev->device);
+ }
+ return idx;
+}
+
+static const char *get_load_fw_name(struct bcm_vk *vk,
+ const struct load_image_entry *entry)
+{
+ const struct firmware *fw;
+ struct device *dev = &vk->pdev->dev;
+ int ret;
+ unsigned long dummy;
+ int i;
+
+ for (i = 0; i < IMG_PER_TYPE_MAX; i++) {
+ fw = NULL;
+ ret = request_partial_firmware_into_buf(&fw,
+ entry->image_name[i],
+ dev, &dummy,
+ sizeof(dummy),
+ 0);
+ release_firmware(fw);
+ if (!ret)
+ return entry->image_name[i];
+ }
+ return NULL;
+}
+
+int bcm_vk_auto_load_all_images(struct bcm_vk *vk)
+{
+ int i, ret = -1;
+ enum soc_idx idx;
+ struct device *dev = &vk->pdev->dev;
+ u32 curr_type;
+ const char *curr_name;
+
+ idx = get_soc_idx(vk);
+ if (idx == VK_IDX_INVALID)
+ goto auto_load_all_exit;
+
+ /* log a message to know the relative loading order */
+ dev_dbg(dev, "Load All for device %d\n", vk->devid);
+
+ for (i = 0; i < NUM_BOOT_STAGES; i++) {
+ curr_type = image_tab[idx][i].image_type;
+ if (bcm_vk_next_boot_image(vk) == curr_type) {
+ curr_name = get_load_fw_name(vk, &image_tab[idx][i]);
+ if (!curr_name) {
+ dev_err(dev, "No suitable firmware exists for type %d",
+ curr_type);
+ ret = -ENOENT;
+ goto auto_load_all_exit;
+ }
+ ret = bcm_vk_load_image_by_type(vk, curr_type,
+ curr_name);
+ dev_info(dev, "Auto load %s, ret %d\n",
+ curr_name, ret);
+
+ if (ret) {
+ dev_err(dev, "Error loading default %s\n",
+ curr_name);
+ goto auto_load_all_exit;
+ }
+ }
+ }
+
+auto_load_all_exit:
+ return ret;
+}
+
+static int bcm_vk_trigger_autoload(struct bcm_vk *vk)
+{
+ if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0)
+ return -EPERM;
+
+ set_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload);
+ queue_work(vk->wq_thread, &vk->wq_work);
+
+ return 0;
+}
+
+/*
+ * deferred work queue for draining and auto download.
+ */
+static void bcm_vk_wq_handler(struct work_struct *work)
+{
+ struct bcm_vk *vk = container_of(work, struct bcm_vk, wq_work);
+ struct device *dev = &vk->pdev->dev;
+ s32 ret;
+
+ /* check wq offload bit map to perform various operations */
+ if (test_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload)) {
+ /* clear bit right the way for notification */
+ clear_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload);
+ bcm_vk_handle_notf(vk);
+ }
+ if (test_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload)) {
+ bcm_vk_auto_load_all_images(vk);
+
+ /*
+ * at the end of operation, clear AUTO bit and pending
+ * bit
+ */
+ clear_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload);
+ clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload);
+ }
+
+ /* next, try to drain */
+ ret = bcm_to_h_msg_dequeue(vk);
+
+ if (ret == 0)
+ dev_dbg(dev, "Spurious trigger for workqueue\n");
+ else if (ret < 0)
+ bcm_vk_blk_drv_access(vk);
+}
+
+static long bcm_vk_load_image(struct bcm_vk *vk,
+ const struct vk_image __user *arg)
+{
+ struct device *dev = &vk->pdev->dev;
+ const char *image_name;
+ struct vk_image image;
+ u32 next_loadable;
+ enum soc_idx idx;
+ int image_idx;
+ int ret = -EPERM;
+
+ if (copy_from_user(&image, arg, sizeof(image)))
+ return -EACCES;
+
+ if ((image.type != VK_IMAGE_TYPE_BOOT1) &&
+ (image.type != VK_IMAGE_TYPE_BOOT2)) {
+ dev_err(dev, "invalid image.type %u\n", image.type);
+ return ret;
+ }
+
+ next_loadable = bcm_vk_next_boot_image(vk);
+ if (next_loadable != image.type) {
+ dev_err(dev, "Next expected image %u, Loading %u\n",
+ next_loadable, image.type);
+ return ret;
+ }
+
+ /*
+ * if something is pending download already. This could only happen
+ * for now when the driver is being loaded, or if someone has issued
+ * another download command in another shell.
+ */
+ if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) {
+ dev_err(dev, "Download operation already pending.\n");
+ return ret;
+ }
+
+ image_name = image.filename;
+ if (image_name[0] == '\0') {
+ /* Use default image name if NULL */
+ idx = get_soc_idx(vk);
+ if (idx == VK_IDX_INVALID)
+ goto err_idx;
+
+ /* Image idx starts with boot1 */
+ image_idx = image.type - VK_IMAGE_TYPE_BOOT1;
+ image_name = get_load_fw_name(vk, &image_tab[idx][image_idx]);
+ if (!image_name) {
+ dev_err(dev, "No suitable image found for type %d",
+ image.type);
+ ret = -ENOENT;
+ goto err_idx;
+ }
+ } else {
+ /* Ensure filename is NULL terminated */
+ image.filename[sizeof(image.filename) - 1] = '\0';
+ }
+ ret = bcm_vk_load_image_by_type(vk, image.type, image_name);
+ dev_info(dev, "Load %s, ret %d\n", image_name, ret);
+err_idx:
+ clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload);
+
+ return ret;
+}
+
+static int bcm_vk_reset_successful(struct bcm_vk *vk)
+{
+ struct device *dev = &vk->pdev->dev;
+ u32 fw_status, reset_reason;
+ int ret = -EAGAIN;
+
+ /*
+ * Reset could be triggered when the card in several state:
+ * i) in bootROM
+ * ii) after boot1
+ * iii) boot2 running
+ *
+ * i) & ii) - no status bits will be updated. If vkboot1
+ * runs automatically after reset, it will update the reason
+ * to be unknown reason
+ * iii) - reboot reason match + deinit done.
+ */
+ fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS);
+ /* immediate exit if interface goes down */
+ if (BCM_VK_INTF_IS_DOWN(fw_status)) {
+ dev_err(dev, "PCIe Intf Down!\n");
+ goto reset_exit;
+ }
+
+ reset_reason = (fw_status & VK_FWSTS_RESET_REASON_MASK);
+ if ((reset_reason == VK_FWSTS_RESET_MBOX_DB) ||
+ (reset_reason == VK_FWSTS_RESET_UNKNOWN))
+ ret = 0;
+
+ /*
+ * if some of the deinit bits are set, but done
+ * bit is not, this is a failure if triggered while boot2 is running
+ */
+ if ((fw_status & VK_FWSTS_DEINIT_TRIGGERED) &&
+ !(fw_status & VK_FWSTS_RESET_DONE))
+ ret = -EAGAIN;
+
+reset_exit:
+ dev_dbg(dev, "FW status = 0x%x ret %d\n", fw_status, ret);
+
+ return ret;
+}
+
+static void bcm_to_v_reset_doorbell(struct bcm_vk *vk, u32 db_val)
+{
+ vkwrite32(vk, db_val, BAR_0, VK_BAR0_RESET_DB_BASE);
+}
+
+static int bcm_vk_trigger_reset(struct bcm_vk *vk)
+{
+ u32 i;
+ u32 value, boot_status;
+ bool is_stdalone, is_boot2;
+ static const u32 bar0_reg_clr_list[] = { BAR_OS_UPTIME,
+ BAR_INTF_VER,
+ BAR_CARD_VOLTAGE,
+ BAR_CARD_TEMPERATURE,
+ BAR_CARD_PWR_AND_THRE };
+
+ /* clean up before pressing the door bell */
+ bcm_vk_drain_msg_on_reset(vk);
+ vkwrite32(vk, 0, BAR_1, VK_BAR1_MSGQ_DEF_RDY);
+ /* make tag '\0' terminated */
+ vkwrite32(vk, 0, BAR_1, VK_BAR1_BOOT1_VER_TAG);
+
+ for (i = 0; i < VK_BAR1_DAUTH_MAX; i++) {
+ vkwrite32(vk, 0, BAR_1, VK_BAR1_DAUTH_STORE_ADDR(i));
+ vkwrite32(vk, 0, BAR_1, VK_BAR1_DAUTH_VALID_ADDR(i));
+ }
+ for (i = 0; i < VK_BAR1_SOTP_REVID_MAX; i++)
+ vkwrite32(vk, 0, BAR_1, VK_BAR1_SOTP_REVID_ADDR(i));
+
+ memset(&vk->card_info, 0, sizeof(vk->card_info));
+ memset(&vk->peerlog_info, 0, sizeof(vk->peerlog_info));
+ memset(&vk->proc_mon_info, 0, sizeof(vk->proc_mon_info));
+ memset(&vk->alert_cnts, 0, sizeof(vk->alert_cnts));
+
+ /*
+ * When boot request fails, the CODE_PUSH_OFFSET stays persistent.
+ * Allowing us to debug the failure. When we call reset,
+ * we should clear CODE_PUSH_OFFSET so ROM does not execute
+ * boot again (and fails again) and instead waits for a new
+ * codepush. And, if previous boot has encountered error, need
+ * to clear the entry values
+ */
+ boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+ if (boot_status & BOOT_ERR_MASK) {
+ dev_info(&vk->pdev->dev,
+ "Card in boot error 0x%x, clear CODEPUSH val\n",
+ boot_status);
+ value = 0;
+ } else {
+ value = vkread32(vk, BAR_0, BAR_CODEPUSH_SBL);
+ value &= CODEPUSH_MASK;
+ }
+ vkwrite32(vk, value, BAR_0, BAR_CODEPUSH_SBL);
+
+ /* special reset handling */
+ is_stdalone = boot_status & BOOT_STDALONE_RUNNING;
+ is_boot2 = (boot_status & BOOT_STATE_MASK) == BOOT2_RUNNING;
+ if (vk->peer_alert.flags & ERR_LOG_RAMDUMP) {
+ /*
+ * if card is in ramdump mode, it is hitting an error. Don't
+ * reset the reboot reason as it will contain valid info that
+ * is important - simply use special reset
+ */
+ vkwrite32(vk, VK_BAR0_RESET_RAMPDUMP, BAR_0, VK_BAR_FWSTS);
+ return VK_BAR0_RESET_RAMPDUMP;
+ } else if (is_stdalone && !is_boot2) {
+ dev_info(&vk->pdev->dev, "Hard reset on Standalone mode");
+ bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_HARD);
+ return VK_BAR0_RESET_DB_HARD;
+ }
+
+ /* reset fw_status with proper reason, and press db */
+ vkwrite32(vk, VK_FWSTS_RESET_MBOX_DB, BAR_0, VK_BAR_FWSTS);
+ bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_SOFT);
+
+ /* clear other necessary registers and alert records */
+ for (i = 0; i < ARRAY_SIZE(bar0_reg_clr_list); i++)
+ vkwrite32(vk, 0, BAR_0, bar0_reg_clr_list[i]);
+ memset(&vk->host_alert, 0, sizeof(vk->host_alert));
+ memset(&vk->peer_alert, 0, sizeof(vk->peer_alert));
+ /* clear 4096 bits of bitmap */
+ bitmap_clear(vk->bmap, 0, VK_MSG_ID_BITMAP_SIZE);
+
+ return 0;
+}
+
+static long bcm_vk_reset(struct bcm_vk *vk, struct vk_reset __user *arg)
+{
+ struct device *dev = &vk->pdev->dev;
+ struct vk_reset reset;
+ int ret = 0;
+ u32 ramdump_reset;
+ int special_reset;
+
+ if (copy_from_user(&reset, arg, sizeof(struct vk_reset)))
+ return -EFAULT;
+
+ /* check if any download is in-progress, if so return error */
+ if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) {
+ dev_err(dev, "Download operation pending - skip reset.\n");
+ return -EPERM;
+ }
+
+ ramdump_reset = vk->peer_alert.flags & ERR_LOG_RAMDUMP;
+ dev_info(dev, "Issue Reset %s\n",
+ ramdump_reset ? "in ramdump mode" : "");
+
+ /*
+ * The following is the sequence of reset:
+ * - send card level graceful shut down
+ * - wait enough time for VK to handle its business, stopping DMA etc
+ * - kill host apps
+ * - Trigger interrupt with DB
+ */
+ bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_GRACEFUL, 0, 0);
+
+ spin_lock(&vk->ctx_lock);
+ if (!vk->reset_pid) {
+ vk->reset_pid = task_pid_nr(current);
+ } else {
+ dev_err(dev, "Reset already launched by process pid %d\n",
+ vk->reset_pid);
+ ret = -EACCES;
+ }
+ spin_unlock(&vk->ctx_lock);
+ if (ret)
+ goto err_exit;
+
+ bcm_vk_blk_drv_access(vk);
+ special_reset = bcm_vk_trigger_reset(vk);
+
+ /*
+ * Wait enough time for card os to deinit
+ * and populate the reset reason.
+ */
+ msleep(BCM_VK_DEINIT_TIME_MS);
+
+ if (special_reset) {
+ /* if it is special ramdump reset, return the type to user */
+ reset.arg2 = special_reset;
+ if (copy_to_user(arg, &reset, sizeof(reset)))
+ ret = -EFAULT;
+ } else {
+ ret = bcm_vk_reset_successful(vk);
+ }
+
+err_exit:
+ clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload);
+ return ret;
+}
+
+static int bcm_vk_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct bcm_vk_ctx *ctx = file->private_data;
+ struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
+ unsigned long pg_size;
+
+ /* only BAR2 is mmap possible, which is bar num 4 due to 64bit */
+#define VK_MMAPABLE_BAR 4
+
+ pg_size = ((pci_resource_len(vk->pdev, VK_MMAPABLE_BAR) - 1)
+ >> PAGE_SHIFT) + 1;
+ if (vma->vm_pgoff + vma_pages(vma) > pg_size)
+ return -EINVAL;
+
+ vma->vm_pgoff += (pci_resource_start(vk->pdev, VK_MMAPABLE_BAR)
+ >> PAGE_SHIFT);
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+ return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+ vma->vm_end - vma->vm_start,
+ vma->vm_page_prot);
+}
+
+static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+ long ret = -EINVAL;
+ struct bcm_vk_ctx *ctx = file->private_data;
+ struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
+ void __user *argp = (void __user *)arg;
+
+ dev_dbg(&vk->pdev->dev,
+ "ioctl, cmd=0x%02x, arg=0x%02lx\n",
+ cmd, arg);
+
+ mutex_lock(&vk->mutex);
+
+ switch (cmd) {
+ case VK_IOCTL_LOAD_IMAGE:
+ ret = bcm_vk_load_image(vk, argp);
+ break;
+
+ case VK_IOCTL_RESET:
+ ret = bcm_vk_reset(vk, argp);
+ break;
+
+ default:
+ break;
+ }
+
+ mutex_unlock(&vk->mutex);
+
+ return ret;
+}
+
+static const struct file_operations bcm_vk_fops = {
+ .owner = THIS_MODULE,
+ .open = bcm_vk_open,
+ .read = bcm_vk_read,
+ .write = bcm_vk_write,
+ .poll = bcm_vk_poll,
+ .release = bcm_vk_release,
+ .mmap = bcm_vk_mmap,
+ .unlocked_ioctl = bcm_vk_ioctl,
+};
+
+static int bcm_vk_on_panic(struct notifier_block *nb,
+ unsigned long e, void *p)
+{
+ struct bcm_vk *vk = container_of(nb, struct bcm_vk, panic_nb);
+
+ bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_HARD);
+
+ return 0;
+}
+
+static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+ int err;
+ int i;
+ int id;
+ int irq;
+ char name[20];
+ struct bcm_vk *vk;
+ struct device *dev = &pdev->dev;
+ struct miscdevice *misc_device;
+ u32 boot_status;
+
+ /* allocate vk structure which is tied to kref for freeing */
+ vk = kzalloc(sizeof(*vk), GFP_KERNEL);
+ if (!vk)
+ return -ENOMEM;
+
+ kref_init(&vk->kref);
+ if (nr_ib_sgl_blk > BCM_VK_IB_SGL_BLK_MAX) {
+ dev_warn(dev, "Inband SGL blk %d limited to max %d\n",
+ nr_ib_sgl_blk, BCM_VK_IB_SGL_BLK_MAX);
+ nr_ib_sgl_blk = BCM_VK_IB_SGL_BLK_MAX;
+ }
+ vk->ib_sgl_size = nr_ib_sgl_blk * VK_MSGQ_BLK_SIZE;
+ mutex_init(&vk->mutex);
+
+ err = pci_enable_device(pdev);
+ if (err) {
+ dev_err(dev, "Cannot enable PCI device\n");
+ goto err_free_exit;
+ }
+ vk->pdev = pci_dev_get(pdev);
+
+ err = pci_request_regions(pdev, DRV_MODULE_NAME);
+ if (err) {
+ dev_err(dev, "Cannot obtain PCI resources\n");
+ goto err_disable_pdev;
+ }
+
+ /* make sure DMA is good */
+ err = dma_set_mask_and_coherent(&pdev->dev,
+ DMA_BIT_MASK(BCM_VK_DMA_BITS));
+ if (err) {
+ dev_err(dev, "failed to set DMA mask\n");
+ goto err_disable_pdev;
+ }
+
+ /* The tdma is a scratch area for some DMA testings. */
+ if (nr_scratch_pages) {
+ vk->tdma_vaddr = dma_alloc_coherent
+ (dev,
+ nr_scratch_pages * PAGE_SIZE,
+ &vk->tdma_addr, GFP_KERNEL);
+ if (!vk->tdma_vaddr) {
+ err = -ENOMEM;
+ goto err_disable_pdev;
+ }
+ }
+
+ pci_set_master(pdev);
+ pci_set_drvdata(pdev, vk);
+
+ irq = pci_alloc_irq_vectors(pdev,
+ VK_MSIX_IRQ_MIN_REQ,
+ VK_MSIX_IRQ_MAX,
+ PCI_IRQ_MSI | PCI_IRQ_MSIX);
+
+ if (irq < VK_MSIX_IRQ_MIN_REQ) {
+ dev_err(dev, "failed to get min %d MSIX interrupts, irq(%d)\n",
+ VK_MSIX_IRQ_MIN_REQ, irq);
+ err = (irq >= 0) ? -EINVAL : irq;
+ goto err_disable_pdev;
+ }
+
+ if (irq != VK_MSIX_IRQ_MAX)
+ dev_warn(dev, "Number of IRQs %d allocated - requested(%d).\n",
+ irq, VK_MSIX_IRQ_MAX);
+
+ for (i = 0; i < MAX_BAR; i++) {
+ /* multiple by 2 for 64 bit BAR mapping */
+ vk->bar[i] = pci_ioremap_bar(pdev, i * 2);
+ if (!vk->bar[i]) {
+ dev_err(dev, "failed to remap BAR%d\n", i);
+ err = -ENOMEM;
+ goto err_iounmap;
+ }
+ }
+
+ for (vk->num_irqs = 0;
+ vk->num_irqs < VK_MSIX_MSGQ_MAX;
+ vk->num_irqs++) {
+ err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs),
+ bcm_vk_msgq_irqhandler,
+ IRQF_SHARED, DRV_MODULE_NAME, vk);
+ if (err) {
+ dev_err(dev, "failed to request msgq IRQ %d for MSIX %d\n",
+ pdev->irq + vk->num_irqs, vk->num_irqs + 1);
+ goto err_irq;
+ }
+ }
+ /* one irq for notification from VK */
+ err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs),
+ bcm_vk_notf_irqhandler,
+ IRQF_SHARED, DRV_MODULE_NAME, vk);
+ if (err) {
+ dev_err(dev, "failed to request notf IRQ %d for MSIX %d\n",
+ pdev->irq + vk->num_irqs, vk->num_irqs + 1);
+ goto err_irq;
+ }
+ vk->num_irqs++;
+
+ for (i = 0;
+ (i < VK_MSIX_TTY_MAX) && (vk->num_irqs < irq);
+ i++, vk->num_irqs++) {
+ err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs),
+ bcm_vk_tty_irqhandler,
+ IRQF_SHARED, DRV_MODULE_NAME, vk);
+ if (err) {
+ dev_err(dev, "failed request tty IRQ %d for MSIX %d\n",
+ pdev->irq + vk->num_irqs, vk->num_irqs + 1);
+ goto err_irq;
+ }
+ bcm_vk_tty_set_irq_enabled(vk, i);
+ }
+
+ id = ida_alloc(&bcm_vk_ida, GFP_KERNEL);
+ if (id < 0) {
+ err = id;
+ dev_err(dev, "unable to get id\n");
+ goto err_irq;
+ }
+
+ vk->devid = id;
+ snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id);
+ misc_device = &vk->miscdev;
+ misc_device->minor = MISC_DYNAMIC_MINOR;
+ misc_device->name = kstrdup(name, GFP_KERNEL);
+ if (!misc_device->name) {
+ err = -ENOMEM;
+ goto err_ida_remove;
+ }
+ misc_device->fops = &bcm_vk_fops,
+
+ err = misc_register(misc_device);
+ if (err) {
+ dev_err(dev, "failed to register device\n");
+ goto err_kfree_name;
+ }
+
+ INIT_WORK(&vk->wq_work, bcm_vk_wq_handler);
+
+ /* create dedicated workqueue */
+ vk->wq_thread = create_singlethread_workqueue(name);
+ if (!vk->wq_thread) {
+ dev_err(dev, "Fail to create workqueue thread\n");
+ err = -ENOMEM;
+ goto err_misc_deregister;
+ }
+
+ err = bcm_vk_msg_init(vk);
+ if (err) {
+ dev_err(dev, "failed to init msg queue info\n");
+ goto err_destroy_workqueue;
+ }
+
+ /* sync other info */
+ bcm_vk_sync_card_info(vk);
+
+ /* register for panic notifier */
+ vk->panic_nb.notifier_call = bcm_vk_on_panic;
+ err = atomic_notifier_chain_register(&panic_notifier_list,
+ &vk->panic_nb);
+ if (err) {
+ dev_err(dev, "Fail to register panic notifier\n");
+ goto err_destroy_workqueue;
+ }
+
+ snprintf(name, sizeof(name), KBUILD_MODNAME ".%d_ttyVK", id);
+ err = bcm_vk_tty_init(vk, name);
+ if (err)
+ goto err_unregister_panic_notifier;
+
+ /*
+ * lets trigger an auto download. We don't want to do it serially here
+ * because at probing time, it is not supposed to block for a long time.
+ */
+ boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+ if (auto_load) {
+ if ((boot_status & BOOT_STATE_MASK) == BROM_RUNNING) {
+ err = bcm_vk_trigger_autoload(vk);
+ if (err)
+ goto err_bcm_vk_tty_exit;
+ } else {
+ dev_err(dev,
+ "Auto-load skipped - BROM not in proper state (0x%x)\n",
+ boot_status);
+ }
+ }
+
+ /* enable hb */
+ bcm_vk_hb_init(vk);
+
+ dev_dbg(dev, "BCM-VK:%u created\n", id);
+
+ return 0;
+
+err_bcm_vk_tty_exit:
+ bcm_vk_tty_exit(vk);
+
+err_unregister_panic_notifier:
+ atomic_notifier_chain_unregister(&panic_notifier_list,
+ &vk->panic_nb);
+
+err_destroy_workqueue:
+ destroy_workqueue(vk->wq_thread);
+
+err_misc_deregister:
+ misc_deregister(misc_device);
+
+err_kfree_name:
+ kfree(misc_device->name);
+ misc_device->name = NULL;
+
+err_ida_remove:
+ ida_free(&bcm_vk_ida, id);
+
+err_irq:
+ for (i = 0; i < vk->num_irqs; i++)
+ devm_free_irq(dev, pci_irq_vector(pdev, i), vk);
+
+ pci_disable_msix(pdev);
+ pci_disable_msi(pdev);
+
+err_iounmap:
+ for (i = 0; i < MAX_BAR; i++) {
+ if (vk->bar[i])
+ pci_iounmap(pdev, vk->bar[i]);
+ }
+ pci_release_regions(pdev);
+
+err_disable_pdev:
+ if (vk->tdma_vaddr)
+ dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE,
+ vk->tdma_vaddr, vk->tdma_addr);
+
+ pci_free_irq_vectors(pdev);
+ pci_disable_device(pdev);
+ pci_dev_put(pdev);
+
+err_free_exit:
+ kfree(vk);
+
+ return err;
+}
+
+void bcm_vk_release_data(struct kref *kref)
+{
+ struct bcm_vk *vk = container_of(kref, struct bcm_vk, kref);
+ struct pci_dev *pdev = vk->pdev;
+
+ dev_dbg(&pdev->dev, "BCM-VK:%d release data 0x%p\n", vk->devid, vk);
+ pci_dev_put(pdev);
+ kfree(vk);
+}
+
+static void bcm_vk_remove(struct pci_dev *pdev)
+{
+ int i;
+ struct bcm_vk *vk = pci_get_drvdata(pdev);
+ struct miscdevice *misc_device = &vk->miscdev;
+
+ bcm_vk_hb_deinit(vk);
+
+ /*
+ * Trigger a reset to card and wait enough time for UCODE to rerun,
+ * which re-initialize the card into its default state.
+ * This ensures when driver is re-enumerated it will start from
+ * a completely clean state.
+ */
+ bcm_vk_trigger_reset(vk);
+ usleep_range(BCM_VK_UCODE_BOOT_US, BCM_VK_UCODE_BOOT_MAX_US);
+
+ /* unregister panic notifier */
+ atomic_notifier_chain_unregister(&panic_notifier_list,
+ &vk->panic_nb);
+
+ bcm_vk_msg_remove(vk);
+ bcm_vk_tty_exit(vk);
+
+ if (vk->tdma_vaddr)
+ dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE,
+ vk->tdma_vaddr, vk->tdma_addr);
+
+ /* remove if name is set which means misc dev registered */
+ if (misc_device->name) {
+ misc_deregister(misc_device);
+ kfree(misc_device->name);
+ ida_free(&bcm_vk_ida, vk->devid);
+ }
+ for (i = 0; i < vk->num_irqs; i++)
+ devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), vk);
+
+ pci_disable_msix(pdev);
+ pci_disable_msi(pdev);
+
+ cancel_work_sync(&vk->wq_work);
+ destroy_workqueue(vk->wq_thread);
+ bcm_vk_tty_wq_exit(vk);
+
+ for (i = 0; i < MAX_BAR; i++) {
+ if (vk->bar[i])
+ pci_iounmap(pdev, vk->bar[i]);
+ }
+
+ dev_dbg(&pdev->dev, "BCM-VK:%d released\n", vk->devid);
+
+ pci_release_regions(pdev);
+ pci_free_irq_vectors(pdev);
+ pci_disable_device(pdev);
+
+ kref_put(&vk->kref, bcm_vk_release_data);
+}
+
+static void bcm_vk_shutdown(struct pci_dev *pdev)
+{
+ struct bcm_vk *vk = pci_get_drvdata(pdev);
+ u32 reg, boot_stat;
+
+ reg = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+ boot_stat = reg & BOOT_STATE_MASK;
+
+ if (boot_stat == BOOT1_RUNNING) {
+ /* simply trigger a reset interrupt to park it */
+ bcm_vk_trigger_reset(vk);
+ } else if (boot_stat == BROM_NOT_RUN) {
+ int err;
+ u16 lnksta;
+
+ /*
+ * The boot status only reflects boot condition since last reset
+ * As ucode will run only once to configure pcie, if multiple
+ * resets happen, we lost track if ucode has run or not.
+ * Here, read the current link speed and use that to
+ * sync up the bootstatus properly so that on reboot-back-up,
+ * it has the proper state to start with autoload
+ */
+ err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
+ if (!err &&
+ (lnksta & PCI_EXP_LNKSTA_CLS) != PCI_EXP_LNKSTA_CLS_2_5GB) {
+ reg |= BROM_STATUS_COMPLETE;
+ vkwrite32(vk, reg, BAR_0, BAR_BOOT_STATUS);
+ }
+ }
+}
+
+static const struct pci_device_id bcm_vk_ids[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VALKYRIE), },
+ { }
+};
+MODULE_DEVICE_TABLE(pci, bcm_vk_ids);
+
+static struct pci_driver pci_driver = {
+ .name = DRV_MODULE_NAME,
+ .id_table = bcm_vk_ids,
+ .probe = bcm_vk_probe,
+ .remove = bcm_vk_remove,
+ .shutdown = bcm_vk_shutdown,
+};
+module_pci_driver(pci_driver);
+
+MODULE_DESCRIPTION("Broadcom VK Host Driver");
+MODULE_AUTHOR("Scott Branden <scott.branden@broadcom.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION("1.0");
diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c
new file mode 100644
index 000000000..3c081504f
--- /dev/null
+++ b/drivers/misc/bcm-vk/bcm_vk_msg.c
@@ -0,0 +1,1352 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#include <linux/delay.h>
+#include <linux/fs.h>
+#include <linux/hash.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/sizes.h>
+#include <linux/spinlock.h>
+#include <linux/timer.h>
+
+#include "bcm_vk.h"
+#include "bcm_vk_msg.h"
+#include "bcm_vk_sg.h"
+
+/* functions to manipulate the transport id in msg block */
+#define BCM_VK_MSG_Q_SHIFT 4
+#define BCM_VK_MSG_Q_MASK 0xF
+#define BCM_VK_MSG_ID_MASK 0xFFF
+
+#define BCM_VK_DMA_DRAIN_MAX_MS 2000
+
+/* number x q_size will be the max number of msg processed per loop */
+#define BCM_VK_MSG_PROC_MAX_LOOP 2
+
+/* module parameter */
+static bool hb_mon = true;
+module_param(hb_mon, bool, 0444);
+MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n");
+static int batch_log = 1;
+module_param(batch_log, int, 0444);
+MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n");
+
+static bool hb_mon_is_on(void)
+{
+ return hb_mon;
+}
+
+static u32 get_q_num(const struct vk_msg_blk *msg)
+{
+ u32 q_num = msg->trans_id & BCM_VK_MSG_Q_MASK;
+
+ if (q_num >= VK_MSGQ_PER_CHAN_MAX)
+ q_num = VK_MSGQ_NUM_DEFAULT;
+ return q_num;
+}
+
+static void set_q_num(struct vk_msg_blk *msg, u32 q_num)
+{
+ u32 trans_q;
+
+ if (q_num >= VK_MSGQ_PER_CHAN_MAX)
+ trans_q = VK_MSGQ_NUM_DEFAULT;
+ else
+ trans_q = q_num;
+
+ msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | trans_q;
+}
+
+static u32 get_msg_id(const struct vk_msg_blk *msg)
+{
+ return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK);
+}
+
+static void set_msg_id(struct vk_msg_blk *msg, u32 val)
+{
+ msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg);
+}
+
+static u32 msgq_inc(const struct bcm_vk_sync_qinfo *qinfo, u32 idx, u32 inc)
+{
+ return ((idx + inc) & qinfo->q_mask);
+}
+
+static
+struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo,
+ u32 idx)
+{
+ return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx);
+}
+
+static u32 msgq_occupied(const struct bcm_vk_msgq __iomem *msgq,
+ const struct bcm_vk_sync_qinfo *qinfo)
+{
+ u32 wr_idx, rd_idx;
+
+ wr_idx = readl_relaxed(&msgq->wr_idx);
+ rd_idx = readl_relaxed(&msgq->rd_idx);
+
+ return ((wr_idx - rd_idx) & qinfo->q_mask);
+}
+
+static
+u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq,
+ const struct bcm_vk_sync_qinfo *qinfo)
+{
+ return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1);
+}
+
+/* number of retries when enqueue message fails before returning EAGAIN */
+#define BCM_VK_H2VK_ENQ_RETRY 10
+#define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50
+
+bool bcm_vk_drv_access_ok(struct bcm_vk *vk)
+{
+ return (!!atomic_read(&vk->msgq_inited));
+}
+
+void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask)
+{
+ struct bcm_vk_alert *alert = &vk->host_alert;
+ unsigned long flags;
+
+ /* use irqsave version as this maybe called inside timer interrupt */
+ spin_lock_irqsave(&vk->host_alert_lock, flags);
+ alert->notfs |= bit_mask;
+ spin_unlock_irqrestore(&vk->host_alert_lock, flags);
+
+ if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0)
+ queue_work(vk->wq_thread, &vk->wq_work);
+}
+
+/*
+ * Heartbeat related defines
+ * The heartbeat from host is a last resort. If stuck condition happens
+ * on the card, firmware is supposed to detect it. Therefore, the heartbeat
+ * values used will be more relaxed on the driver, which need to be bigger
+ * than the watchdog timeout on the card. The watchdog timeout on the card
+ * is 20s, with a jitter of 2s => 22s. We use a value of 27s here.
+ */
+#define BCM_VK_HB_TIMER_S 3
+#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ)
+#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S)
+
+static void bcm_vk_hb_poll(struct timer_list *t)
+{
+ u32 uptime_s;
+ struct bcm_vk_hb_ctrl *hb = container_of(t, struct bcm_vk_hb_ctrl,
+ timer);
+ struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);
+
+ if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) {
+ /* read uptime from register and compare */
+ uptime_s = vkread32(vk, BAR_0, BAR_OS_UPTIME);
+
+ if (uptime_s == hb->last_uptime)
+ hb->lost_cnt++;
+ else /* reset to avoid accumulation */
+ hb->lost_cnt = 0;
+
+ dev_dbg(&vk->pdev->dev, "Last uptime %d current %d, lost %d\n",
+ hb->last_uptime, uptime_s, hb->lost_cnt);
+
+ /*
+ * if the interface goes down without any activity, a value
+ * of 0xFFFFFFFF will be continuously read, and the detection
+ * will be happened eventually.
+ */
+ hb->last_uptime = uptime_s;
+ } else {
+ /* reset heart beat lost cnt */
+ hb->lost_cnt = 0;
+ }
+
+ /* next, check if heartbeat exceeds limit */
+ if (hb->lost_cnt > BCM_VK_HB_LOST_MAX) {
+ dev_err(&vk->pdev->dev, "Heartbeat Misses %d times, %d s!\n",
+ BCM_VK_HB_LOST_MAX,
+ BCM_VK_HB_LOST_MAX * BCM_VK_HB_TIMER_S);
+
+ bcm_vk_blk_drv_access(vk);
+ bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL);
+ }
+ /* re-arm timer */
+ mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE);
+}
+
+void bcm_vk_hb_init(struct bcm_vk *vk)
+{
+ struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl;
+
+ timer_setup(&hb->timer, bcm_vk_hb_poll, 0);
+ mod_timer(&hb->timer, jiffies + BCM_VK_HB_TIMER_VALUE);
+}
+
+void bcm_vk_hb_deinit(struct bcm_vk *vk)
+{
+ struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl;
+
+ del_timer(&hb->timer);
+}
+
+static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,
+ unsigned int start,
+ unsigned int nbits)
+{
+ spin_lock(&vk->msg_id_lock);
+ bitmap_clear(vk->bmap, start, nbits);
+ spin_unlock(&vk->msg_id_lock);
+}
+
+/*
+ * allocate a ctx per file struct
+ */
+static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid)
+{
+ u32 i;
+ struct bcm_vk_ctx *ctx = NULL;
+ u32 hash_idx = hash_32(pid, VK_PID_HT_SHIFT_BIT);
+
+ spin_lock(&vk->ctx_lock);
+
+ /* check if it is in reset, if so, don't allow */
+ if (vk->reset_pid) {
+ dev_err(&vk->pdev->dev,
+ "No context allowed during reset by pid %d\n",
+ vk->reset_pid);
+
+ goto in_reset_exit;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) {
+ if (!vk->ctx[i].in_use) {
+ vk->ctx[i].in_use = true;
+ ctx = &vk->ctx[i];
+ break;
+ }
+ }
+
+ if (!ctx) {
+ dev_err(&vk->pdev->dev, "All context in use\n");
+
+ goto all_in_use_exit;
+ }
+
+ /* set the pid and insert it to hash table */
+ ctx->pid = pid;
+ ctx->hash_idx = hash_idx;
+ list_add_tail(&ctx->node, &vk->pid_ht[hash_idx].head);
+
+ /* increase kref */
+ kref_get(&vk->kref);
+
+ /* clear counter */
+ atomic_set(&ctx->pend_cnt, 0);
+ atomic_set(&ctx->dma_cnt, 0);
+ init_waitqueue_head(&ctx->rd_wq);
+
+all_in_use_exit:
+in_reset_exit:
+ spin_unlock(&vk->ctx_lock);
+
+ return ctx;
+}
+
+static u16 bcm_vk_get_msg_id(struct bcm_vk *vk)
+{
+ u16 rc = VK_MSG_ID_OVERFLOW;
+ u16 test_bit_count = 0;
+
+ spin_lock(&vk->msg_id_lock);
+ while (test_bit_count < (VK_MSG_ID_BITMAP_SIZE - 1)) {
+ /*
+ * first time come in this loop, msg_id will be 0
+ * and the first one tested will be 1. We skip
+ * VK_SIMPLEX_MSG_ID (0) for one way host2vk
+ * communication
+ */
+ vk->msg_id++;
+ if (vk->msg_id == VK_MSG_ID_BITMAP_SIZE)
+ vk->msg_id = 1;
+
+ if (test_bit(vk->msg_id, vk->bmap)) {
+ test_bit_count++;
+ continue;
+ }
+ rc = vk->msg_id;
+ bitmap_set(vk->bmap, vk->msg_id, 1);
+ break;
+ }
+ spin_unlock(&vk->msg_id_lock);
+
+ return rc;
+}
+
+static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx)
+{
+ u32 idx;
+ u32 hash_idx;
+ pid_t pid;
+ struct bcm_vk_ctx *entry;
+ int count = 0;
+
+ if (!ctx) {
+ dev_err(&vk->pdev->dev, "NULL context detected\n");
+ return -EINVAL;
+ }
+ idx = ctx->idx;
+ pid = ctx->pid;
+
+ spin_lock(&vk->ctx_lock);
+
+ if (!vk->ctx[idx].in_use) {
+ dev_err(&vk->pdev->dev, "context[%d] not in use!\n", idx);
+ } else {
+ vk->ctx[idx].in_use = false;
+ vk->ctx[idx].miscdev = NULL;
+
+ /* Remove it from hash list and see if it is the last one. */
+ list_del(&ctx->node);
+ hash_idx = ctx->hash_idx;
+ list_for_each_entry(entry, &vk->pid_ht[hash_idx].head, node) {
+ if (entry->pid == pid)
+ count++;
+ }
+ }
+
+ spin_unlock(&vk->ctx_lock);
+
+ return count;
+}
+
+static void bcm_vk_free_wkent(struct device *dev, struct bcm_vk_wkent *entry)
+{
+ int proc_cnt;
+
+ bcm_vk_sg_free(dev, entry->dma, VK_DMA_MAX_ADDRS, &proc_cnt);
+ if (proc_cnt)
+ atomic_dec(&entry->ctx->dma_cnt);
+
+ kfree(entry->to_h_msg);
+ kfree(entry);
+}
+
+static void bcm_vk_drain_all_pend(struct device *dev,
+ struct bcm_vk_msg_chan *chan,
+ struct bcm_vk_ctx *ctx)
+{
+ u32 num;
+ struct bcm_vk_wkent *entry, *tmp;
+ struct bcm_vk *vk;
+ struct list_head del_q;
+
+ if (ctx)
+ vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
+
+ INIT_LIST_HEAD(&del_q);
+ spin_lock(&chan->pendq_lock);
+ for (num = 0; num < chan->q_nr; num++) {
+ list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) {
+ if ((!ctx) || (entry->ctx->idx == ctx->idx)) {
+ list_move_tail(&entry->node, &del_q);
+ }
+ }
+ }
+ spin_unlock(&chan->pendq_lock);
+
+ /* batch clean up */
+ num = 0;
+ list_for_each_entry_safe(entry, tmp, &del_q, node) {
+ list_del(&entry->node);
+ num++;
+ if (ctx) {
+ struct vk_msg_blk *msg;
+ int bit_set;
+ bool responded;
+ u32 msg_id;
+
+ /* if it is specific ctx, log for any stuck */
+ msg = entry->to_v_msg;
+ msg_id = get_msg_id(msg);
+ bit_set = test_bit(msg_id, vk->bmap);
+ responded = entry->to_h_msg ? true : false;
+ if (num <= batch_log)
+ dev_info(dev,
+ "Drained: fid %u size %u msg 0x%x(seq-%x) ctx 0x%x[fd-%d] args:[0x%x 0x%x] resp %s, bmap %d\n",
+ msg->function_id, msg->size,
+ msg_id, entry->seq_num,
+ msg->context_id, entry->ctx->idx,
+ msg->cmd, msg->arg,
+ responded ? "T" : "F", bit_set);
+ if (responded)
+ atomic_dec(&ctx->pend_cnt);
+ else if (bit_set)
+ bcm_vk_msgid_bitmap_clear(vk, msg_id, 1);
+ }
+ bcm_vk_free_wkent(dev, entry);
+ }
+ if (num && ctx)
+ dev_info(dev, "Total drained items %d [fd-%d]\n",
+ num, ctx->idx);
+}
+
+void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk)
+{
+ bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL);
+ bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL);
+}
+
+/*
+ * Function to sync up the messages queue info that is provided by BAR1
+ */
+int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync)
+{
+ struct bcm_vk_msgq __iomem *msgq;
+ struct device *dev = &vk->pdev->dev;
+ u32 msgq_off;
+ u32 num_q;
+ struct bcm_vk_msg_chan *chan_list[] = {&vk->to_v_msg_chan,
+ &vk->to_h_msg_chan};
+ struct bcm_vk_msg_chan *chan;
+ int i, j;
+ int ret = 0;
+
+ /*
+ * If the driver is loaded at startup where vk OS is not up yet,
+ * the msgq-info may not be available until a later time. In
+ * this case, we skip and the sync function is supposed to be
+ * called again.
+ */
+ if (!bcm_vk_msgq_marker_valid(vk)) {
+ dev_info(dev, "BAR1 msgq marker not initialized.\n");
+ return -EAGAIN;
+ }
+
+ msgq_off = vkread32(vk, BAR_1, VK_BAR1_MSGQ_CTRL_OFF);
+
+ /* each side is always half the total */
+ num_q = vkread32(vk, BAR_1, VK_BAR1_MSGQ_NR) / 2;
+ if (!num_q || (num_q > VK_MSGQ_PER_CHAN_MAX)) {
+ dev_err(dev,
+ "Advertised msgq %d error - max %d allowed\n",
+ num_q, VK_MSGQ_PER_CHAN_MAX);
+ return -EINVAL;
+ }
+
+ vk->to_v_msg_chan.q_nr = num_q;
+ vk->to_h_msg_chan.q_nr = num_q;
+
+ /* first msgq location */
+ msgq = vk->bar[BAR_1] + msgq_off;
+
+ /*
+ * if this function is called when it is already inited,
+ * something is wrong
+ */
+ if (bcm_vk_drv_access_ok(vk) && !force_sync) {
+ dev_err(dev, "Msgq info already in sync\n");
+ return -EPERM;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(chan_list); i++) {
+ chan = chan_list[i];
+ memset(chan->sync_qinfo, 0, sizeof(chan->sync_qinfo));
+
+ for (j = 0; j < num_q; j++) {
+ struct bcm_vk_sync_qinfo *qinfo;
+ u32 msgq_start;
+ u32 msgq_size;
+ u32 msgq_nxt;
+ u32 msgq_db_offset, q_db_offset;
+
+ chan->msgq[j] = msgq;
+ msgq_start = readl_relaxed(&msgq->start);
+ msgq_size = readl_relaxed(&msgq->size);
+ msgq_nxt = readl_relaxed(&msgq->nxt);
+ msgq_db_offset = readl_relaxed(&msgq->db_offset);
+ q_db_offset = (msgq_db_offset & ((1 << DB_SHIFT) - 1));
+ if (q_db_offset == (~msgq_db_offset >> DB_SHIFT))
+ msgq_db_offset = q_db_offset;
+ else
+ /* fall back to default */
+ msgq_db_offset = VK_BAR0_Q_DB_BASE(j);
+
+ dev_info(dev,
+ "MsgQ[%d] type %d num %d, @ 0x%x, db_offset 0x%x rd_idx %d wr_idx %d, size %d, nxt 0x%x\n",
+ j,
+ readw_relaxed(&msgq->type),
+ readw_relaxed(&msgq->num),
+ msgq_start,
+ msgq_db_offset,
+ readl_relaxed(&msgq->rd_idx),
+ readl_relaxed(&msgq->wr_idx),
+ msgq_size,
+ msgq_nxt);
+
+ qinfo = &chan->sync_qinfo[j];
+ /* formulate and record static info */
+ qinfo->q_start = vk->bar[BAR_1] + msgq_start;
+ qinfo->q_size = msgq_size;
+ /* set low threshold as 50% or 1/2 */
+ qinfo->q_low = qinfo->q_size >> 1;
+ qinfo->q_mask = qinfo->q_size - 1;
+ qinfo->q_db_offset = msgq_db_offset;
+
+ msgq++;
+ }
+ }
+ atomic_set(&vk->msgq_inited, 1);
+
+ return ret;
+}
+
+static int bcm_vk_msg_chan_init(struct bcm_vk_msg_chan *chan)
+{
+ u32 i;
+
+ mutex_init(&chan->msgq_mutex);
+ spin_lock_init(&chan->pendq_lock);
+ for (i = 0; i < VK_MSGQ_MAX_NR; i++)
+ INIT_LIST_HEAD(&chan->pendq[i]);
+
+ return 0;
+}
+
+static void bcm_vk_append_pendq(struct bcm_vk_msg_chan *chan, u16 q_num,
+ struct bcm_vk_wkent *entry)
+{
+ struct bcm_vk_ctx *ctx;
+
+ spin_lock(&chan->pendq_lock);
+ list_add_tail(&entry->node, &chan->pendq[q_num]);
+ if (entry->to_h_msg) {
+ ctx = entry->ctx;
+ atomic_inc(&ctx->pend_cnt);
+ wake_up_interruptible(&ctx->rd_wq);
+ }
+ spin_unlock(&chan->pendq_lock);
+}
+
+static u32 bcm_vk_append_ib_sgl(struct bcm_vk *vk,
+ struct bcm_vk_wkent *entry,
+ struct _vk_data *data,
+ unsigned int num_planes)
+{
+ unsigned int i;
+ unsigned int item_cnt = 0;
+ struct device *dev = &vk->pdev->dev;
+ struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
+ struct vk_msg_blk *msg = &entry->to_v_msg[0];
+ struct bcm_vk_msgq __iomem *msgq;
+ struct bcm_vk_sync_qinfo *qinfo;
+ u32 ib_sgl_size = 0;
+ u8 *buf = (u8 *)&entry->to_v_msg[entry->to_v_blks];
+ u32 avail;
+ u32 q_num;
+
+ /* check if high watermark is hit, and if so, skip */
+ q_num = get_q_num(msg);
+ msgq = chan->msgq[q_num];
+ qinfo = &chan->sync_qinfo[q_num];
+ avail = msgq_avail_space(msgq, qinfo);
+ if (avail < qinfo->q_low) {
+ dev_dbg(dev, "Skip inserting inband SGL, [0x%x/0x%x]\n",
+ avail, qinfo->q_size);
+ return 0;
+ }
+
+ for (i = 0; i < num_planes; i++) {
+ if (data[i].address &&
+ (ib_sgl_size + data[i].size) <= vk->ib_sgl_size) {
+ item_cnt++;
+ memcpy(buf, entry->dma[i].sglist, data[i].size);
+ ib_sgl_size += data[i].size;
+ buf += data[i].size;
+ }
+ }
+
+ dev_dbg(dev, "Num %u sgl items appended, size 0x%x, room 0x%x\n",
+ item_cnt, ib_sgl_size, vk->ib_sgl_size);
+
+ /* round up size */
+ ib_sgl_size = (ib_sgl_size + VK_MSGQ_BLK_SIZE - 1)
+ >> VK_MSGQ_BLK_SZ_SHIFT;
+
+ return ib_sgl_size;
+}
+
+void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val)
+{
+ struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
+ struct bcm_vk_sync_qinfo *qinfo = &chan->sync_qinfo[q_num];
+
+ vkwrite32(vk, db_val, BAR_0, qinfo->q_db_offset);
+}
+
+static int bcm_to_v_msg_enqueue(struct bcm_vk *vk, struct bcm_vk_wkent *entry)
+{
+ static u32 seq_num;
+ struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
+ struct device *dev = &vk->pdev->dev;
+ struct vk_msg_blk *src = &entry->to_v_msg[0];
+
+ struct vk_msg_blk __iomem *dst;
+ struct bcm_vk_msgq __iomem *msgq;
+ struct bcm_vk_sync_qinfo *qinfo;
+ u32 q_num = get_q_num(src);
+ u32 wr_idx; /* local copy */
+ u32 i;
+ u32 avail;
+ u32 retry;
+
+ if (entry->to_v_blks != src->size + 1) {
+ dev_err(dev, "number of blks %d not matching %d MsgId[0x%x]: func %d ctx 0x%x\n",
+ entry->to_v_blks,
+ src->size + 1,
+ get_msg_id(src),
+ src->function_id,
+ src->context_id);
+ return -EMSGSIZE;
+ }
+
+ msgq = chan->msgq[q_num];
+ qinfo = &chan->sync_qinfo[q_num];
+
+ mutex_lock(&chan->msgq_mutex);
+
+ avail = msgq_avail_space(msgq, qinfo);
+
+ /* if not enough space, return EAGAIN and let app handles it */
+ retry = 0;
+ while ((avail < entry->to_v_blks) &&
+ (retry++ < BCM_VK_H2VK_ENQ_RETRY)) {
+ mutex_unlock(&chan->msgq_mutex);
+
+ msleep(BCM_VK_H2VK_ENQ_RETRY_DELAY_MS);
+ mutex_lock(&chan->msgq_mutex);
+ avail = msgq_avail_space(msgq, qinfo);
+ }
+ if (retry > BCM_VK_H2VK_ENQ_RETRY) {
+ mutex_unlock(&chan->msgq_mutex);
+ return -EAGAIN;
+ }
+
+ /* at this point, mutex is taken and there is enough space */
+ entry->seq_num = seq_num++; /* update debug seq number */
+ wr_idx = readl_relaxed(&msgq->wr_idx);
+
+ if (wr_idx >= qinfo->q_size) {
+ dev_crit(dev, "Invalid wr_idx 0x%x => max 0x%x!",
+ wr_idx, qinfo->q_size);
+ bcm_vk_blk_drv_access(vk);
+ bcm_vk_set_host_alert(vk, ERR_LOG_HOST_PCIE_DWN);
+ goto idx_err;
+ }
+
+ dst = msgq_blk_addr(qinfo, wr_idx);
+ for (i = 0; i < entry->to_v_blks; i++) {
+ memcpy_toio(dst, src, sizeof(*dst));
+
+ src++;
+ wr_idx = msgq_inc(qinfo, wr_idx, 1);
+ dst = msgq_blk_addr(qinfo, wr_idx);
+ }
+
+ /* flush the write pointer */
+ writel(wr_idx, &msgq->wr_idx);
+
+ /* log new info for debugging */
+ dev_dbg(dev,
+ "MsgQ[%d] [Rd Wr] = [%d %d] blks inserted %d - Q = [u-%d a-%d]/%d\n",
+ readl_relaxed(&msgq->num),
+ readl_relaxed(&msgq->rd_idx),
+ wr_idx,
+ entry->to_v_blks,
+ msgq_occupied(msgq, qinfo),
+ msgq_avail_space(msgq, qinfo),
+ readl_relaxed(&msgq->size));
+ /*
+ * press door bell based on queue number. 1 is added to the wr_idx
+ * to avoid the value of 0 appearing on the VK side to distinguish
+ * from initial value.
+ */
+ bcm_to_v_q_doorbell(vk, q_num, wr_idx + 1);
+idx_err:
+ mutex_unlock(&chan->msgq_mutex);
+ return 0;
+}
+
+int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type,
+ const pid_t pid, const u32 q_num)
+{
+ int rc = 0;
+ struct bcm_vk_wkent *entry;
+ struct device *dev = &vk->pdev->dev;
+
+ /*
+ * check if the marker is still good. Sometimes, the PCIe interface may
+ * have gone done, and if so and we ship down thing based on broken
+ * values, kernel may panic.
+ */
+ if (!bcm_vk_msgq_marker_valid(vk)) {
+ dev_info(dev, "PCIe comm chan - invalid marker (0x%x)!\n",
+ vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY));
+ return -EINVAL;
+ }
+
+ entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL);
+ if (!entry)
+ return -ENOMEM;
+
+ /* fill up necessary data */
+ entry->to_v_msg[0].function_id = VK_FID_SHUTDOWN;
+ set_q_num(&entry->to_v_msg[0], q_num);
+ set_msg_id(&entry->to_v_msg[0], VK_SIMPLEX_MSG_ID);
+ entry->to_v_blks = 1; /* always 1 block */
+
+ entry->to_v_msg[0].cmd = shut_type;
+ entry->to_v_msg[0].arg = pid;
+
+ rc = bcm_to_v_msg_enqueue(vk, entry);
+ if (rc)
+ dev_err(dev,
+ "Sending shutdown message to q %d for pid %d fails.\n",
+ get_q_num(&entry->to_v_msg[0]), pid);
+
+ kfree(entry);
+
+ return rc;
+}
+
+static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid,
+ const u32 q_num)
+{
+ int rc = 0;
+ struct device *dev = &vk->pdev->dev;
+
+ /*
+ * don't send down or do anything if message queue is not initialized
+ * and if it is the reset session, clear it.
+ */
+ if (!bcm_vk_drv_access_ok(vk)) {
+ if (vk->reset_pid == pid)
+ vk->reset_pid = 0;
+ return -EPERM;
+ }
+
+ dev_dbg(dev, "No more sessions, shut down pid %d\n", pid);
+
+ /* only need to do it if it is not the reset process */
+ if (vk->reset_pid != pid)
+ rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num);
+ else
+ /* put reset_pid to 0 if it is exiting last session */
+ vk->reset_pid = 0;
+
+ return rc;
+}
+
+static struct bcm_vk_wkent *bcm_vk_dequeue_pending(struct bcm_vk *vk,
+ struct bcm_vk_msg_chan *chan,
+ u16 q_num,
+ u16 msg_id)
+{
+ struct bcm_vk_wkent *entry = NULL, *iter;
+
+ spin_lock(&chan->pendq_lock);
+ list_for_each_entry(iter, &chan->pendq[q_num], node) {
+ if (get_msg_id(&iter->to_v_msg[0]) == msg_id) {
+ list_del(&iter->node);
+ entry = iter;
+ bcm_vk_msgid_bitmap_clear(vk, msg_id, 1);
+ break;
+ }
+ }
+ spin_unlock(&chan->pendq_lock);
+ return entry;
+}
+
+s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk)
+{
+ struct device *dev = &vk->pdev->dev;
+ struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan;
+ struct vk_msg_blk *data;
+ struct vk_msg_blk __iomem *src;
+ struct vk_msg_blk *dst;
+ struct bcm_vk_msgq __iomem *msgq;
+ struct bcm_vk_sync_qinfo *qinfo;
+ struct bcm_vk_wkent *entry;
+ u32 rd_idx, wr_idx;
+ u32 q_num, msg_id, j;
+ u32 num_blks;
+ s32 total = 0;
+ int cnt = 0;
+ int msg_processed = 0;
+ int max_msg_to_process;
+ bool exit_loop;
+
+ /*
+ * drain all the messages from the queues, and find its pending
+ * entry in the to_v queue, based on msg_id & q_num, and move the
+ * entry to the to_h pending queue, waiting for user space
+ * program to extract
+ */
+ mutex_lock(&chan->msgq_mutex);
+
+ for (q_num = 0; q_num < chan->q_nr; q_num++) {
+ msgq = chan->msgq[q_num];
+ qinfo = &chan->sync_qinfo[q_num];
+ max_msg_to_process = BCM_VK_MSG_PROC_MAX_LOOP * qinfo->q_size;
+
+ rd_idx = readl_relaxed(&msgq->rd_idx);
+ wr_idx = readl_relaxed(&msgq->wr_idx);
+ msg_processed = 0;
+ exit_loop = false;
+ while ((rd_idx != wr_idx) && !exit_loop) {
+ u8 src_size;
+
+ /*
+ * Make a local copy and get pointer to src blk
+ * The rd_idx is masked before getting the pointer to
+ * avoid out of bound access in case the interface goes
+ * down. It will end up pointing to the last block in
+ * the buffer, but subsequent src->size check would be
+ * able to catch this.
+ */
+ src = msgq_blk_addr(qinfo, rd_idx & qinfo->q_mask);
+ src_size = readb(&src->size);
+
+ if ((rd_idx >= qinfo->q_size) ||
+ (src_size > (qinfo->q_size - 1))) {
+ dev_crit(dev,
+ "Invalid rd_idx 0x%x or size 0x%x => max 0x%x!",
+ rd_idx, src_size, qinfo->q_size);
+ bcm_vk_blk_drv_access(vk);
+ bcm_vk_set_host_alert(vk,
+ ERR_LOG_HOST_PCIE_DWN);
+ goto idx_err;
+ }
+
+ num_blks = src_size + 1;
+ data = kzalloc(num_blks * VK_MSGQ_BLK_SIZE, GFP_KERNEL);
+ if (data) {
+ /* copy messages and linearize it */
+ dst = data;
+ for (j = 0; j < num_blks; j++) {
+ memcpy_fromio(dst, src, sizeof(*dst));
+
+ dst++;
+ rd_idx = msgq_inc(qinfo, rd_idx, 1);
+ src = msgq_blk_addr(qinfo, rd_idx);
+ }
+ total++;
+ } else {
+ /*
+ * if we could not allocate memory in kernel,
+ * that is fatal.
+ */
+ dev_crit(dev, "Kernel mem allocation failure.\n");
+ total = -ENOMEM;
+ goto idx_err;
+ }
+
+ /* flush rd pointer after a message is dequeued */
+ writel(rd_idx, &msgq->rd_idx);
+
+ /* log new info for debugging */
+ dev_dbg(dev,
+ "MsgQ[%d] [Rd Wr] = [%d %d] blks extracted %d - Q = [u-%d a-%d]/%d\n",
+ readl_relaxed(&msgq->num),
+ rd_idx,
+ wr_idx,
+ num_blks,
+ msgq_occupied(msgq, qinfo),
+ msgq_avail_space(msgq, qinfo),
+ readl_relaxed(&msgq->size));
+
+ /*
+ * No need to search if it is an autonomous one-way
+ * message from driver, as these messages do not bear
+ * a to_v pending item. Currently, only the shutdown
+ * message falls into this category.
+ */
+ if (data->function_id == VK_FID_SHUTDOWN) {
+ kfree(data);
+ continue;
+ }
+
+ msg_id = get_msg_id(data);
+ /* lookup original message in to_v direction */
+ entry = bcm_vk_dequeue_pending(vk,
+ &vk->to_v_msg_chan,
+ q_num,
+ msg_id);
+
+ /*
+ * if there is message to does not have prior send,
+ * this is the location to add here
+ */
+ if (entry) {
+ entry->to_h_blks = num_blks;
+ entry->to_h_msg = data;
+ bcm_vk_append_pendq(&vk->to_h_msg_chan,
+ q_num, entry);
+
+ } else {
+ if (cnt++ < batch_log)
+ dev_info(dev,
+ "Could not find MsgId[0x%x] for resp func %d bmap %d\n",
+ msg_id, data->function_id,
+ test_bit(msg_id, vk->bmap));
+ kfree(data);
+ }
+ /* Fetch wr_idx to handle more back-to-back events */
+ wr_idx = readl(&msgq->wr_idx);
+
+ /*
+ * cap the max so that even we try to handle more back-to-back events,
+ * so that it won't hold CPU too long or in case rd/wr idexes are
+ * corrupted which triggers infinite looping.
+ */
+ if (++msg_processed >= max_msg_to_process) {
+ dev_warn(dev, "Q[%d] Per loop processing exceeds %d\n",
+ q_num, max_msg_to_process);
+ exit_loop = true;
+ }
+ }
+ }
+idx_err:
+ mutex_unlock(&chan->msgq_mutex);
+ dev_dbg(dev, "total %d drained from queues\n", total);
+
+ return total;
+}
+
+/*
+ * init routine for all required data structures
+ */
+static int bcm_vk_data_init(struct bcm_vk *vk)
+{
+ int i;
+
+ spin_lock_init(&vk->ctx_lock);
+ for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) {
+ vk->ctx[i].in_use = false;
+ vk->ctx[i].idx = i; /* self identity */
+ vk->ctx[i].miscdev = NULL;
+ }
+ spin_lock_init(&vk->msg_id_lock);
+ spin_lock_init(&vk->host_alert_lock);
+ vk->msg_id = 0;
+
+ /* initialize hash table */
+ for (i = 0; i < VK_PID_HT_SZ; i++)
+ INIT_LIST_HEAD(&vk->pid_ht[i].head);
+
+ return 0;
+}
+
+irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id)
+{
+ struct bcm_vk *vk = dev_id;
+
+ if (!bcm_vk_drv_access_ok(vk)) {
+ dev_err(&vk->pdev->dev,
+ "Interrupt %d received when msgq not inited\n", irq);
+ goto skip_schedule_work;
+ }
+
+ queue_work(vk->wq_thread, &vk->wq_work);
+
+skip_schedule_work:
+ return IRQ_HANDLED;
+}
+
+int bcm_vk_open(struct inode *inode, struct file *p_file)
+{
+ struct bcm_vk_ctx *ctx;
+ struct miscdevice *miscdev = (struct miscdevice *)p_file->private_data;
+ struct bcm_vk *vk = container_of(miscdev, struct bcm_vk, miscdev);
+ struct device *dev = &vk->pdev->dev;
+ int rc = 0;
+
+ /* get a context and set it up for file */
+ ctx = bcm_vk_get_ctx(vk, task_tgid_nr(current));
+ if (!ctx) {
+ dev_err(dev, "Error allocating context\n");
+ rc = -ENOMEM;
+ } else {
+ /*
+ * set up context and replace private data with context for
+ * other methods to use. Reason for the context is because
+ * it is allowed for multiple sessions to open the sysfs, and
+ * for each file open, when upper layer query the response,
+ * only those that are tied to a specific open should be
+ * returned. The context->idx will be used for such binding
+ */
+ ctx->miscdev = miscdev;
+ p_file->private_data = ctx;
+ dev_dbg(dev, "ctx_returned with idx %d, pid %d\n",
+ ctx->idx, ctx->pid);
+ }
+ return rc;
+}
+
+ssize_t bcm_vk_read(struct file *p_file,
+ char __user *buf,
+ size_t count,
+ loff_t *f_pos)
+{
+ ssize_t rc = -ENOMSG;
+ struct bcm_vk_ctx *ctx = p_file->private_data;
+ struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk,
+ miscdev);
+ struct device *dev = &vk->pdev->dev;
+ struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan;
+ struct bcm_vk_wkent *entry = NULL, *iter;
+ u32 q_num;
+ u32 rsp_length;
+
+ if (!bcm_vk_drv_access_ok(vk))
+ return -EPERM;
+
+ dev_dbg(dev, "Buf count %zu\n", count);
+
+ /*
+ * search through the pendq on the to_h chan, and return only those
+ * that belongs to the same context. Search is always from the high to
+ * the low priority queues
+ */
+ spin_lock(&chan->pendq_lock);
+ for (q_num = 0; q_num < chan->q_nr; q_num++) {
+ list_for_each_entry(iter, &chan->pendq[q_num], node) {
+ if (iter->ctx->idx == ctx->idx) {
+ if (count >=
+ (iter->to_h_blks * VK_MSGQ_BLK_SIZE)) {
+ list_del(&iter->node);
+ atomic_dec(&ctx->pend_cnt);
+ entry = iter;
+ } else {
+ /* buffer not big enough */
+ rc = -EMSGSIZE;
+ }
+ goto read_loop_exit;
+ }
+ }
+ }
+read_loop_exit:
+ spin_unlock(&chan->pendq_lock);
+
+ if (entry) {
+ /* retrieve the passed down msg_id */
+ set_msg_id(&entry->to_h_msg[0], entry->usr_msg_id);
+ rsp_length = entry->to_h_blks * VK_MSGQ_BLK_SIZE;
+ if (copy_to_user(buf, entry->to_h_msg, rsp_length) == 0)
+ rc = rsp_length;
+
+ bcm_vk_free_wkent(dev, entry);
+ } else if (rc == -EMSGSIZE) {
+ struct vk_msg_blk tmp_msg = entry->to_h_msg[0];
+
+ /*
+ * in this case, return just the first block, so
+ * that app knows what size it is looking for.
+ */
+ set_msg_id(&tmp_msg, entry->usr_msg_id);
+ tmp_msg.size = entry->to_h_blks - 1;
+ if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) {
+ dev_err(dev, "Error return 1st block in -EMSGSIZE\n");
+ rc = -EFAULT;
+ }
+ }
+ return rc;
+}
+
+ssize_t bcm_vk_write(struct file *p_file,
+ const char __user *buf,
+ size_t count,
+ loff_t *f_pos)
+{
+ ssize_t rc;
+ struct bcm_vk_ctx *ctx = p_file->private_data;
+ struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk,
+ miscdev);
+ struct bcm_vk_msgq __iomem *msgq;
+ struct device *dev = &vk->pdev->dev;
+ struct bcm_vk_wkent *entry;
+ u32 sgl_extra_blks;
+ u32 q_num;
+ u32 msg_size;
+ u32 msgq_size;
+
+ if (!bcm_vk_drv_access_ok(vk))
+ return -EPERM;
+
+ dev_dbg(dev, "Msg count %zu\n", count);
+
+ /* first, do sanity check where count should be multiple of basic blk */
+ if (count & (VK_MSGQ_BLK_SIZE - 1)) {
+ dev_err(dev, "Failure with size %zu not multiple of %zu\n",
+ count, VK_MSGQ_BLK_SIZE);
+ rc = -EINVAL;
+ goto write_err;
+ }
+
+ /* allocate the work entry + buffer for size count and inband sgl */
+ entry = kzalloc(sizeof(*entry) + count + vk->ib_sgl_size,
+ GFP_KERNEL);
+ if (!entry) {
+ rc = -ENOMEM;
+ goto write_err;
+ }
+
+ /* now copy msg from user space, and then formulate the work entry */
+ if (copy_from_user(&entry->to_v_msg[0], buf, count)) {
+ rc = -EFAULT;
+ goto write_free_ent;
+ }
+
+ entry->to_v_blks = count >> VK_MSGQ_BLK_SZ_SHIFT;
+ entry->ctx = ctx;
+
+ /* do a check on the blk size which could not exceed queue space */
+ q_num = get_q_num(&entry->to_v_msg[0]);
+ msgq = vk->to_v_msg_chan.msgq[q_num];
+ msgq_size = readl_relaxed(&msgq->size);
+ if (entry->to_v_blks + (vk->ib_sgl_size >> VK_MSGQ_BLK_SZ_SHIFT)
+ > (msgq_size - 1)) {
+ dev_err(dev, "Blk size %d exceed max queue size allowed %d\n",
+ entry->to_v_blks, msgq_size - 1);
+ rc = -EINVAL;
+ goto write_free_ent;
+ }
+
+ /* Use internal message id */
+ entry->usr_msg_id = get_msg_id(&entry->to_v_msg[0]);
+ rc = bcm_vk_get_msg_id(vk);
+ if (rc == VK_MSG_ID_OVERFLOW) {
+ dev_err(dev, "msg_id overflow\n");
+ rc = -EOVERFLOW;
+ goto write_free_ent;
+ }
+ set_msg_id(&entry->to_v_msg[0], rc);
+ ctx->q_num = q_num;
+
+ dev_dbg(dev,
+ "[Q-%d]Message ctx id %d, usr_msg_id 0x%x sent msg_id 0x%x\n",
+ ctx->q_num, ctx->idx, entry->usr_msg_id,
+ get_msg_id(&entry->to_v_msg[0]));
+
+ if (entry->to_v_msg[0].function_id == VK_FID_TRANS_BUF) {
+ /* Convert any pointers to sg list */
+ unsigned int num_planes;
+ int dir;
+ struct _vk_data *data;
+
+ /*
+ * check if we are in reset, if so, no buffer transfer is
+ * allowed and return error.
+ */
+ if (vk->reset_pid) {
+ dev_dbg(dev, "No Transfer allowed during reset, pid %d.\n",
+ ctx->pid);
+ rc = -EACCES;
+ goto write_free_msgid;
+ }
+
+ num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK;
+ if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD)
+ dir = DMA_FROM_DEVICE;
+ else
+ dir = DMA_TO_DEVICE;
+
+ /* Calculate vk_data location */
+ /* Go to end of the message */
+ msg_size = entry->to_v_msg[0].size;
+ if (msg_size > entry->to_v_blks) {
+ rc = -EMSGSIZE;
+ goto write_free_msgid;
+ }
+
+ data = (struct _vk_data *)&entry->to_v_msg[msg_size + 1];
+
+ /* Now back up to the start of the pointers */
+ data -= num_planes;
+
+ /* Convert user addresses to DMA SG List */
+ rc = bcm_vk_sg_alloc(dev, entry->dma, dir, data, num_planes);
+ if (rc)
+ goto write_free_msgid;
+
+ atomic_inc(&ctx->dma_cnt);
+ /* try to embed inband sgl */
+ sgl_extra_blks = bcm_vk_append_ib_sgl(vk, entry, data,
+ num_planes);
+ entry->to_v_blks += sgl_extra_blks;
+ entry->to_v_msg[0].size += sgl_extra_blks;
+ } else if (entry->to_v_msg[0].function_id == VK_FID_INIT &&
+ entry->to_v_msg[0].context_id == VK_NEW_CTX) {
+ /*
+ * Init happens in 2 stages, only the first stage contains the
+ * pid that needs translating.
+ */
+ pid_t org_pid, pid;
+
+ /*
+ * translate the pid into the unique host space as user
+ * may run sessions inside containers or process
+ * namespaces.
+ */
+#define VK_MSG_PID_MASK 0xffffff00
+#define VK_MSG_PID_SH 8
+ org_pid = (entry->to_v_msg[0].arg & VK_MSG_PID_MASK)
+ >> VK_MSG_PID_SH;
+
+ pid = task_tgid_nr(current);
+ entry->to_v_msg[0].arg =
+ (entry->to_v_msg[0].arg & ~VK_MSG_PID_MASK) |
+ (pid << VK_MSG_PID_SH);
+ if (org_pid != pid)
+ dev_dbg(dev, "In PID 0x%x(%d), converted PID 0x%x(%d)\n",
+ org_pid, org_pid, pid, pid);
+ }
+
+ /*
+ * store work entry to pending queue until a response is received.
+ * This needs to be done before enqueuing the message
+ */
+ bcm_vk_append_pendq(&vk->to_v_msg_chan, q_num, entry);
+
+ rc = bcm_to_v_msg_enqueue(vk, entry);
+ if (rc) {
+ dev_err(dev, "Fail to enqueue msg to to_v queue\n");
+
+ /* remove message from pending list */
+ entry = bcm_vk_dequeue_pending
+ (vk,
+ &vk->to_v_msg_chan,
+ q_num,
+ get_msg_id(&entry->to_v_msg[0]));
+ goto write_free_ent;
+ }
+
+ return count;
+
+write_free_msgid:
+ bcm_vk_msgid_bitmap_clear(vk, get_msg_id(&entry->to_v_msg[0]), 1);
+write_free_ent:
+ kfree(entry);
+write_err:
+ return rc;
+}
+
+__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait)
+{
+ __poll_t ret = 0;
+ int cnt;
+ struct bcm_vk_ctx *ctx = p_file->private_data;
+ struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
+ struct device *dev = &vk->pdev->dev;
+
+ poll_wait(p_file, &ctx->rd_wq, wait);
+
+ cnt = atomic_read(&ctx->pend_cnt);
+ if (cnt) {
+ ret = (__force __poll_t)(POLLIN | POLLRDNORM);
+ if (cnt < 0) {
+ dev_err(dev, "Error cnt %d, setting back to 0", cnt);
+ atomic_set(&ctx->pend_cnt, 0);
+ }
+ }
+
+ return ret;
+}
+
+int bcm_vk_release(struct inode *inode, struct file *p_file)
+{
+ int ret;
+ struct bcm_vk_ctx *ctx = p_file->private_data;
+ struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
+ struct device *dev = &vk->pdev->dev;
+ pid_t pid = ctx->pid;
+ int dma_cnt;
+ unsigned long timeout, start_time;
+
+ /*
+ * if there are outstanding DMA transactions, need to delay long enough
+ * to ensure that the card side would have stopped touching the host buffer
+ * and its SGL list. A race condition could happen if the host app is killed
+ * abruptly, eg kill -9, while some DMA transfer orders are still inflight.
+ * Nothing could be done except for a delay as host side is running in a
+ * completely async fashion.
+ */
+ start_time = jiffies;
+ timeout = start_time + msecs_to_jiffies(BCM_VK_DMA_DRAIN_MAX_MS);
+ do {
+ if (time_after(jiffies, timeout)) {
+ dev_warn(dev, "%d dma still pending for [fd-%d] pid %d\n",
+ dma_cnt, ctx->idx, pid);
+ break;
+ }
+ dma_cnt = atomic_read(&ctx->dma_cnt);
+ cpu_relax();
+ cond_resched();
+ } while (dma_cnt);
+ dev_dbg(dev, "Draining for [fd-%d] pid %d - delay %d ms\n",
+ ctx->idx, pid, jiffies_to_msecs(jiffies - start_time));
+
+ bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, ctx);
+ bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, ctx);
+
+ ret = bcm_vk_free_ctx(vk, ctx);
+ if (ret == 0)
+ ret = bcm_vk_handle_last_sess(vk, pid, ctx->q_num);
+ else
+ ret = 0;
+
+ kref_put(&vk->kref, bcm_vk_release_data);
+
+ return ret;
+}
+
+int bcm_vk_msg_init(struct bcm_vk *vk)
+{
+ struct device *dev = &vk->pdev->dev;
+ int ret;
+
+ if (bcm_vk_data_init(vk)) {
+ dev_err(dev, "Error initializing internal data structures\n");
+ return -EINVAL;
+ }
+
+ if (bcm_vk_msg_chan_init(&vk->to_v_msg_chan) ||
+ bcm_vk_msg_chan_init(&vk->to_h_msg_chan)) {
+ dev_err(dev, "Error initializing communication channel\n");
+ return -EIO;
+ }
+
+ /* read msgq info if ready */
+ ret = bcm_vk_sync_msgq(vk, false);
+ if (ret && (ret != -EAGAIN)) {
+ dev_err(dev, "Error reading comm msg Q info\n");
+ return -EIO;
+ }
+
+ return 0;
+}
+
+void bcm_vk_msg_remove(struct bcm_vk *vk)
+{
+ bcm_vk_blk_drv_access(vk);
+
+ /* drain all pending items */
+ bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL);
+ bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL);
+}
+
diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.h b/drivers/misc/bcm-vk/bcm_vk_msg.h
new file mode 100644
index 000000000..56784c889
--- /dev/null
+++ b/drivers/misc/bcm-vk/bcm_vk_msg.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#ifndef BCM_VK_MSG_H
+#define BCM_VK_MSG_H
+
+#include <uapi/linux/misc/bcm_vk.h>
+#include "bcm_vk_sg.h"
+
+/* Single message queue control structure */
+struct bcm_vk_msgq {
+ u16 type; /* queue type */
+ u16 num; /* queue number */
+ u32 start; /* offset in BAR1 where the queue memory starts */
+
+ u32 rd_idx; /* read idx */
+ u32 wr_idx; /* write idx */
+
+ u32 size; /*
+ * size, which is in number of 16byte blocks,
+ * to align with the message data structure.
+ */
+ u32 nxt; /*
+ * nxt offset to the next msg queue struct.
+ * This is to provide flexibity for alignment purposes.
+ */
+
+/* Least significant 16 bits in below field hold doorbell register offset */
+#define DB_SHIFT 16
+
+ u32 db_offset; /* queue doorbell register offset in BAR0 */
+
+ u32 rsvd;
+};
+
+/*
+ * Structure to record static info from the msgq sync. We keep local copy
+ * for some of these variables for both performance + checking purpose.
+ */
+struct bcm_vk_sync_qinfo {
+ void __iomem *q_start;
+ u32 q_size;
+ u32 q_mask;
+ u32 q_low;
+ u32 q_db_offset;
+};
+
+#define VK_MSGQ_MAX_NR 4 /* Maximum number of message queues */
+
+/*
+ * message block - basic unit in the message where a message's size is always
+ * N x sizeof(basic_block)
+ */
+struct vk_msg_blk {
+ u8 function_id;
+#define VK_FID_TRANS_BUF 5
+#define VK_FID_SHUTDOWN 8
+#define VK_FID_INIT 9
+ u8 size; /* size of the message in number of vk_msg_blk's */
+ u16 trans_id; /* transport id, queue & msg_id */
+ u32 context_id;
+#define VK_NEW_CTX 0
+ u32 cmd;
+#define VK_CMD_PLANES_MASK 0x000f /* number of planes to up/download */
+#define VK_CMD_UPLOAD 0x0400 /* memory transfer to vk */
+#define VK_CMD_DOWNLOAD 0x0500 /* memory transfer from vk */
+#define VK_CMD_MASK 0x0f00 /* command mask */
+ u32 arg;
+};
+
+/* vk_msg_blk is 16 bytes fixed */
+#define VK_MSGQ_BLK_SIZE (sizeof(struct vk_msg_blk))
+/* shift for fast division of basic msg blk size */
+#define VK_MSGQ_BLK_SZ_SHIFT 4
+
+/* use msg_id 0 for any simplex host2vk communication */
+#define VK_SIMPLEX_MSG_ID 0
+
+/* context per session opening of sysfs */
+struct bcm_vk_ctx {
+ struct list_head node; /* use for linkage in Hash Table */
+ unsigned int idx;
+ bool in_use;
+ pid_t pid;
+ u32 hash_idx;
+ u32 q_num; /* queue number used by the stream */
+ struct miscdevice *miscdev;
+ atomic_t pend_cnt; /* number of items pending to be read from host */
+ atomic_t dma_cnt; /* any dma transaction outstanding */
+ wait_queue_head_t rd_wq;
+};
+
+/* pid hash table entry */
+struct bcm_vk_ht_entry {
+ struct list_head head;
+};
+
+#define VK_DMA_MAX_ADDRS 4 /* Max 4 DMA Addresses */
+/* structure for house keeping a single work entry */
+struct bcm_vk_wkent {
+ struct list_head node; /* for linking purpose */
+ struct bcm_vk_ctx *ctx;
+
+ /* Store up to 4 dma pointers */
+ struct bcm_vk_dma dma[VK_DMA_MAX_ADDRS];
+
+ u32 to_h_blks; /* response */
+ struct vk_msg_blk *to_h_msg;
+
+ /*
+ * put the to_v_msg at the end so that we could simply append to_v msg
+ * to the end of the allocated block
+ */
+ u32 usr_msg_id;
+ u32 to_v_blks;
+ u32 seq_num;
+ struct vk_msg_blk to_v_msg[];
+};
+
+/* queue stats counters */
+struct bcm_vk_qs_cnts {
+ u32 cnt; /* general counter, used to limit output */
+ u32 acc_sum;
+ u32 max_occ; /* max during a sampling period */
+ u32 max_abs; /* the abs max since reset */
+};
+
+/* control channel structure for either to_v or to_h communication */
+struct bcm_vk_msg_chan {
+ u32 q_nr;
+ /* Mutex to access msgq */
+ struct mutex msgq_mutex;
+ /* pointing to BAR locations */
+ struct bcm_vk_msgq __iomem *msgq[VK_MSGQ_MAX_NR];
+ /* Spinlock to access pending queue */
+ spinlock_t pendq_lock;
+ /* for temporary storing pending items, one for each queue */
+ struct list_head pendq[VK_MSGQ_MAX_NR];
+ /* static queue info from the sync */
+ struct bcm_vk_sync_qinfo sync_qinfo[VK_MSGQ_MAX_NR];
+};
+
+/* totol number of message q allowed by the driver */
+#define VK_MSGQ_PER_CHAN_MAX 3
+#define VK_MSGQ_NUM_DEFAULT (VK_MSGQ_PER_CHAN_MAX - 1)
+
+/* total number of supported ctx, 32 ctx each for 5 components */
+#define VK_CMPT_CTX_MAX (32 * 5)
+
+/* hash table defines to store the opened FDs */
+#define VK_PID_HT_SHIFT_BIT 7 /* 128 */
+#define VK_PID_HT_SZ BIT(VK_PID_HT_SHIFT_BIT)
+
+/* The following are offsets of DDR info provided by the vk card */
+#define VK_BAR0_SEG_SIZE (4 * SZ_1K) /* segment size for BAR0 */
+
+/* shutdown types supported */
+#define VK_SHUTDOWN_PID 1
+#define VK_SHUTDOWN_GRACEFUL 2
+
+#endif
diff --git a/drivers/misc/bcm-vk/bcm_vk_sg.c b/drivers/misc/bcm-vk/bcm_vk_sg.c
new file mode 100644
index 000000000..2e9daaf3e
--- /dev/null
+++ b/drivers/misc/bcm-vk/bcm_vk_sg.c
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+#include <linux/dma-mapping.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/pgtable.h>
+#include <linux/vmalloc.h>
+
+#include <asm/page.h>
+#include <asm/unaligned.h>
+
+#include <uapi/linux/misc/bcm_vk.h>
+
+#include "bcm_vk.h"
+#include "bcm_vk_msg.h"
+#include "bcm_vk_sg.h"
+
+/*
+ * Valkyrie has a hardware limitation of 16M transfer size.
+ * So limit the SGL chunks to 16M.
+ */
+#define BCM_VK_MAX_SGL_CHUNK SZ_16M
+
+static int bcm_vk_dma_alloc(struct device *dev,
+ struct bcm_vk_dma *dma,
+ int dir,
+ struct _vk_data *vkdata);
+static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma);
+
+/* Uncomment to dump SGLIST */
+/* #define BCM_VK_DUMP_SGLIST */
+
+static int bcm_vk_dma_alloc(struct device *dev,
+ struct bcm_vk_dma *dma,
+ int direction,
+ struct _vk_data *vkdata)
+{
+ dma_addr_t addr, sg_addr;
+ int err;
+ int i;
+ int offset;
+ u32 size;
+ u32 remaining_size;
+ u32 transfer_size;
+ u64 data;
+ unsigned long first, last;
+ struct _vk_data *sgdata;
+
+ /* Get 64-bit user address */
+ data = get_unaligned(&vkdata->address);
+
+ /* offset into first page */
+ offset = offset_in_page(data);
+
+ /* Calculate number of pages */
+ first = (data & PAGE_MASK) >> PAGE_SHIFT;
+ last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT;
+ dma->nr_pages = last - first + 1;
+
+ /* Allocate DMA pages */
+ dma->pages = kmalloc_array(dma->nr_pages,
+ sizeof(struct page *),
+ GFP_KERNEL);
+ if (!dma->pages)
+ return -ENOMEM;
+
+ dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n",
+ data, vkdata->size, dma->nr_pages);
+
+ dma->direction = direction;
+
+ /* Get user pages into memory */
+ err = get_user_pages_fast(data & PAGE_MASK,
+ dma->nr_pages,
+ direction == DMA_FROM_DEVICE,
+ dma->pages);
+ if (err != dma->nr_pages) {
+ dma->nr_pages = (err >= 0) ? err : 0;
+ dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",
+ err, dma->nr_pages);
+ return err < 0 ? err : -EINVAL;
+ }
+
+ /* Max size of sg list is 1 per mapped page + fields at start */
+ dma->sglen = (dma->nr_pages * sizeof(*sgdata)) +
+ (sizeof(u32) * SGLIST_VKDATA_START);
+
+ /* Allocate sglist */
+ dma->sglist = dma_alloc_coherent(dev,
+ dma->sglen,
+ &dma->handle,
+ GFP_KERNEL);
+ if (!dma->sglist)
+ return -ENOMEM;
+
+ dma->sglist[SGLIST_NUM_SG] = 0;
+ dma->sglist[SGLIST_TOTALSIZE] = vkdata->size;
+ remaining_size = vkdata->size;
+ sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
+
+ /* Map all pages into DMA */
+ size = min_t(size_t, PAGE_SIZE - offset, remaining_size);
+ remaining_size -= size;
+ sg_addr = dma_map_page(dev,
+ dma->pages[0],
+ offset,
+ size,
+ dma->direction);
+ transfer_size = size;
+ if (unlikely(dma_mapping_error(dev, sg_addr))) {
+ __free_page(dma->pages[0]);
+ return -EIO;
+ }
+
+ for (i = 1; i < dma->nr_pages; i++) {
+ size = min_t(size_t, PAGE_SIZE, remaining_size);
+ remaining_size -= size;
+ addr = dma_map_page(dev,
+ dma->pages[i],
+ 0,
+ size,
+ dma->direction);
+ if (unlikely(dma_mapping_error(dev, addr))) {
+ __free_page(dma->pages[i]);
+ return -EIO;
+ }
+
+ /*
+ * Compress SG list entry when pages are contiguous
+ * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK
+ */
+ if ((addr == (sg_addr + transfer_size)) &&
+ ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) {
+ /* pages are contiguous, add to same sg entry */
+ transfer_size += size;
+ } else {
+ /* pages are not contiguous, write sg entry */
+ sgdata->size = transfer_size;
+ put_unaligned(sg_addr, (u64 *)&sgdata->address);
+ dma->sglist[SGLIST_NUM_SG]++;
+
+ /* start new sg entry */
+ sgdata++;
+ sg_addr = addr;
+ transfer_size = size;
+ }
+ }
+ /* Write last sg list entry */
+ sgdata->size = transfer_size;
+ put_unaligned(sg_addr, (u64 *)&sgdata->address);
+ dma->sglist[SGLIST_NUM_SG]++;
+
+ /* Update pointers and size field to point to sglist */
+ put_unaligned((u64)dma->handle, &vkdata->address);
+ vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) +
+ (sizeof(u32) * SGLIST_VKDATA_START);
+
+#ifdef BCM_VK_DUMP_SGLIST
+ dev_dbg(dev,
+ "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n",
+ (u64)dma->sglist,
+ dma->handle,
+ dma->sglen,
+ vkdata->size);
+ for (i = 0; i < vkdata->size / sizeof(u32); i++)
+ dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]);
+#endif
+
+ return 0;
+}
+
+int bcm_vk_sg_alloc(struct device *dev,
+ struct bcm_vk_dma *dma,
+ int dir,
+ struct _vk_data *vkdata,
+ int num)
+{
+ int i;
+ int rc = -EINVAL;
+
+ /* Convert user addresses to DMA SG List */
+ for (i = 0; i < num; i++) {
+ if (vkdata[i].size && vkdata[i].address) {
+ /*
+ * If both size and address are non-zero
+ * then DMA alloc.
+ */
+ rc = bcm_vk_dma_alloc(dev,
+ &dma[i],
+ dir,
+ &vkdata[i]);
+ } else if (vkdata[i].size ||
+ vkdata[i].address) {
+ /*
+ * If one of size and address are zero
+ * there is a problem.
+ */
+ dev_err(dev,
+ "Invalid vkdata %x 0x%x 0x%llx\n",
+ i, vkdata[i].size, vkdata[i].address);
+ rc = -EINVAL;
+ } else {
+ /*
+ * If size and address are both zero
+ * don't convert, but return success.
+ */
+ rc = 0;
+ }
+
+ if (rc)
+ goto fail_alloc;
+ }
+ return rc;
+
+fail_alloc:
+ while (i > 0) {
+ i--;
+ if (dma[i].sglist)
+ bcm_vk_dma_free(dev, &dma[i]);
+ }
+ return rc;
+}
+
+static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma)
+{
+ dma_addr_t addr;
+ int i;
+ int num_sg;
+ u32 size;
+ struct _vk_data *vkdata;
+
+ dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen);
+
+ /* Unmap all pages in the sglist */
+ num_sg = dma->sglist[SGLIST_NUM_SG];
+ vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
+ for (i = 0; i < num_sg; i++) {
+ size = vkdata[i].size;
+ addr = get_unaligned(&vkdata[i].address);
+
+ dma_unmap_page(dev, addr, size, dma->direction);
+ }
+
+ /* Free allocated sglist */
+ dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle);
+
+ /* Release lock on all pages */
+ for (i = 0; i < dma->nr_pages; i++)
+ put_page(dma->pages[i]);
+
+ /* Free allocated dma pages */
+ kfree(dma->pages);
+ dma->sglist = NULL;
+
+ return 0;
+}
+
+int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,
+ int *proc_cnt)
+{
+ int i;
+
+ *proc_cnt = 0;
+ /* Unmap and free all pages and sglists */
+ for (i = 0; i < num; i++) {
+ if (dma[i].sglist) {
+ bcm_vk_dma_free(dev, &dma[i]);
+ *proc_cnt += 1;
+ }
+ }
+
+ return 0;
+}
diff --git a/drivers/misc/bcm-vk/bcm_vk_sg.h b/drivers/misc/bcm-vk/bcm_vk_sg.h
new file mode 100644
index 000000000..81b3d0976
--- /dev/null
+++ b/drivers/misc/bcm-vk/bcm_vk_sg.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#ifndef BCM_VK_SG_H
+#define BCM_VK_SG_H
+
+#include <linux/dma-mapping.h>
+
+struct bcm_vk_dma {
+ /* for userland buffer */
+ struct page **pages;
+ int nr_pages;
+
+ /* common */
+ dma_addr_t handle;
+ /*
+ * sglist is of the following LE format
+ * [U32] num_sg = number of sg addresses (N)
+ * [U32] totalsize = totalsize of data being transferred in sglist
+ * [U32] size[0] = size of data in address0
+ * [U32] addr_l[0] = lower 32-bits of address0
+ * [U32] addr_h[0] = higher 32-bits of address0
+ * ..
+ * [U32] size[N-1] = size of data in addressN-1
+ * [U32] addr_l[N-1] = lower 32-bits of addressN-1
+ * [U32] addr_h[N-1] = higher 32-bits of addressN-1
+ */
+ u32 *sglist;
+#define SGLIST_NUM_SG 0
+#define SGLIST_TOTALSIZE 1
+#define SGLIST_VKDATA_START 2
+
+ int sglen; /* Length (bytes) of sglist */
+ int direction;
+};
+
+struct _vk_data {
+ u32 size; /* data size in bytes */
+ u64 address; /* Pointer to data */
+} __packed;
+
+/*
+ * Scatter-gather DMA buffer API.
+ *
+ * These functions provide a simple way to create a page list and a
+ * scatter-gather list from userspace address and map the memory
+ * for DMA operation.
+ */
+int bcm_vk_sg_alloc(struct device *dev,
+ struct bcm_vk_dma *dma,
+ int dir,
+ struct _vk_data *vkdata,
+ int num);
+
+int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,
+ int *proc_cnt);
+
+#endif
+
diff --git a/drivers/misc/bcm-vk/bcm_vk_tty.c b/drivers/misc/bcm-vk/bcm_vk_tty.c
new file mode 100644
index 000000000..6669625ba
--- /dev/null
+++ b/drivers/misc/bcm-vk/bcm_vk_tty.c
@@ -0,0 +1,339 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#include <linux/tty.h>
+#include <linux/tty_driver.h>
+#include <linux/tty_flip.h>
+
+#include "bcm_vk.h"
+
+/* TTYVK base offset is 0x30000 into BAR1 */
+#define BAR1_TTYVK_BASE_OFFSET 0x300000
+/* Each TTYVK channel (TO or FROM) is 0x10000 */
+#define BAR1_TTYVK_CHAN_OFFSET 0x100000
+/* Each TTYVK channel has TO and FROM, hence the * 2 */
+#define BAR1_TTYVK_BASE(index) (BAR1_TTYVK_BASE_OFFSET + \
+ ((index) * BAR1_TTYVK_CHAN_OFFSET * 2))
+/* TO TTYVK channel base comes before FROM for each index */
+#define TO_TTYK_BASE(index) BAR1_TTYVK_BASE(index)
+#define FROM_TTYK_BASE(index) (BAR1_TTYVK_BASE(index) + \
+ BAR1_TTYVK_CHAN_OFFSET)
+
+struct bcm_vk_tty_chan {
+ u32 reserved;
+ u32 size;
+ u32 wr;
+ u32 rd;
+ u32 *data;
+};
+
+#define VK_BAR_CHAN(v, DIR, e) ((v)->DIR##_offset \
+ + offsetof(struct bcm_vk_tty_chan, e))
+#define VK_BAR_CHAN_SIZE(v, DIR) VK_BAR_CHAN(v, DIR, size)
+#define VK_BAR_CHAN_WR(v, DIR) VK_BAR_CHAN(v, DIR, wr)
+#define VK_BAR_CHAN_RD(v, DIR) VK_BAR_CHAN(v, DIR, rd)
+#define VK_BAR_CHAN_DATA(v, DIR, off) (VK_BAR_CHAN(v, DIR, data) + (off))
+
+#define VK_BAR0_REGSEG_TTY_DB_OFFSET 0x86c
+
+/* Poll every 1/10 of second - temp hack till we use MSI interrupt */
+#define SERIAL_TIMER_VALUE (HZ / 10)
+
+static void bcm_vk_tty_poll(struct timer_list *t)
+{
+ struct bcm_vk *vk = from_timer(vk, t, serial_timer);
+
+ queue_work(vk->tty_wq_thread, &vk->tty_wq_work);
+ mod_timer(&vk->serial_timer, jiffies + SERIAL_TIMER_VALUE);
+}
+
+irqreturn_t bcm_vk_tty_irqhandler(int irq, void *dev_id)
+{
+ struct bcm_vk *vk = dev_id;
+
+ queue_work(vk->tty_wq_thread, &vk->tty_wq_work);
+
+ return IRQ_HANDLED;
+}
+
+static void bcm_vk_tty_wq_handler(struct work_struct *work)
+{
+ struct bcm_vk *vk = container_of(work, struct bcm_vk, tty_wq_work);
+ struct bcm_vk_tty *vktty;
+ int card_status;
+ int count;
+ unsigned char c;
+ int i;
+ int wr;
+
+ card_status = vkread32(vk, BAR_0, BAR_CARD_STATUS);
+ if (BCM_VK_INTF_IS_DOWN(card_status))
+ return;
+
+ for (i = 0; i < BCM_VK_NUM_TTY; i++) {
+ count = 0;
+ /* Check the card status that the tty channel is ready */
+ if ((card_status & BIT(i)) == 0)
+ continue;
+
+ vktty = &vk->tty[i];
+
+ /* Don't increment read index if tty app is closed */
+ if (!vktty->is_opened)
+ continue;
+
+ /* Fetch the wr offset in buffer from VK */
+ wr = vkread32(vk, BAR_1, VK_BAR_CHAN_WR(vktty, from));
+
+ /* safe to ignore until bar read gives proper size */
+ if (vktty->from_size == 0)
+ continue;
+
+ if (wr >= vktty->from_size) {
+ dev_err(&vk->pdev->dev,
+ "ERROR: wq handler ttyVK%d wr:0x%x > 0x%x\n",
+ i, wr, vktty->from_size);
+ /* Need to signal and close device in this case */
+ continue;
+ }
+
+ /*
+ * Simple read of circular buffer and
+ * insert into tty flip buffer
+ */
+ while (vk->tty[i].rd != wr) {
+ c = vkread8(vk, BAR_1,
+ VK_BAR_CHAN_DATA(vktty, from, vktty->rd));
+ vktty->rd++;
+ if (vktty->rd >= vktty->from_size)
+ vktty->rd = 0;
+ tty_insert_flip_char(&vktty->port, c, TTY_NORMAL);
+ count++;
+ }
+
+ if (count) {
+ tty_flip_buffer_push(&vktty->port);
+
+ /* Update read offset from shadow register to card */
+ vkwrite32(vk, vktty->rd, BAR_1,
+ VK_BAR_CHAN_RD(vktty, from));
+ }
+ }
+}
+
+static int bcm_vk_tty_open(struct tty_struct *tty, struct file *file)
+{
+ int card_status;
+ struct bcm_vk *vk;
+ struct bcm_vk_tty *vktty;
+ int index;
+
+ /* initialize the pointer in case something fails */
+ tty->driver_data = NULL;
+
+ vk = (struct bcm_vk *)dev_get_drvdata(tty->dev);
+ index = tty->index;
+
+ if (index >= BCM_VK_NUM_TTY)
+ return -EINVAL;
+
+ vktty = &vk->tty[index];
+
+ vktty->pid = task_pid_nr(current);
+ vktty->to_offset = TO_TTYK_BASE(index);
+ vktty->from_offset = FROM_TTYK_BASE(index);
+
+ /* Do not allow tty device to be opened if tty on card not ready */
+ card_status = vkread32(vk, BAR_0, BAR_CARD_STATUS);
+ if (BCM_VK_INTF_IS_DOWN(card_status) || ((card_status & BIT(index)) == 0))
+ return -EBUSY;
+
+ /*
+ * Get shadow registers of the buffer sizes and the "to" write offset
+ * and "from" read offset
+ */
+ vktty->to_size = vkread32(vk, BAR_1, VK_BAR_CHAN_SIZE(vktty, to));
+ vktty->wr = vkread32(vk, BAR_1, VK_BAR_CHAN_WR(vktty, to));
+ vktty->from_size = vkread32(vk, BAR_1, VK_BAR_CHAN_SIZE(vktty, from));
+ vktty->rd = vkread32(vk, BAR_1, VK_BAR_CHAN_RD(vktty, from));
+ vktty->is_opened = true;
+
+ if (tty->count == 1 && !vktty->irq_enabled) {
+ timer_setup(&vk->serial_timer, bcm_vk_tty_poll, 0);
+ mod_timer(&vk->serial_timer, jiffies + SERIAL_TIMER_VALUE);
+ }
+ return 0;
+}
+
+static void bcm_vk_tty_close(struct tty_struct *tty, struct file *file)
+{
+ struct bcm_vk *vk = dev_get_drvdata(tty->dev);
+
+ if (tty->index >= BCM_VK_NUM_TTY)
+ return;
+
+ vk->tty[tty->index].is_opened = false;
+
+ if (tty->count == 1)
+ del_timer_sync(&vk->serial_timer);
+}
+
+static void bcm_vk_tty_doorbell(struct bcm_vk *vk, u32 db_val)
+{
+ vkwrite32(vk, db_val, BAR_0,
+ VK_BAR0_REGSEG_DB_BASE + VK_BAR0_REGSEG_TTY_DB_OFFSET);
+}
+
+static int bcm_vk_tty_write(struct tty_struct *tty,
+ const unsigned char *buffer,
+ int count)
+{
+ int index;
+ struct bcm_vk *vk;
+ struct bcm_vk_tty *vktty;
+ int i;
+
+ index = tty->index;
+ vk = dev_get_drvdata(tty->dev);
+ vktty = &vk->tty[index];
+
+ /* Simple write each byte to circular buffer */
+ for (i = 0; i < count; i++) {
+ vkwrite8(vk, buffer[i], BAR_1,
+ VK_BAR_CHAN_DATA(vktty, to, vktty->wr));
+ vktty->wr++;
+ if (vktty->wr >= vktty->to_size)
+ vktty->wr = 0;
+ }
+ /* Update write offset from shadow register to card */
+ vkwrite32(vk, vktty->wr, BAR_1, VK_BAR_CHAN_WR(vktty, to));
+ bcm_vk_tty_doorbell(vk, 0);
+
+ return count;
+}
+
+static unsigned int bcm_vk_tty_write_room(struct tty_struct *tty)
+{
+ struct bcm_vk *vk = dev_get_drvdata(tty->dev);
+
+ return vk->tty[tty->index].to_size - 1;
+}
+
+static const struct tty_operations serial_ops = {
+ .open = bcm_vk_tty_open,
+ .close = bcm_vk_tty_close,
+ .write = bcm_vk_tty_write,
+ .write_room = bcm_vk_tty_write_room,
+};
+
+int bcm_vk_tty_init(struct bcm_vk *vk, char *name)
+{
+ int i;
+ int err;
+ struct tty_driver *tty_drv;
+ struct device *dev = &vk->pdev->dev;
+
+ tty_drv = tty_alloc_driver
+ (BCM_VK_NUM_TTY,
+ TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV);
+ if (IS_ERR(tty_drv))
+ return PTR_ERR(tty_drv);
+
+ /* Save struct tty_driver for uninstalling the device */
+ vk->tty_drv = tty_drv;
+
+ /* initialize the tty driver */
+ tty_drv->driver_name = KBUILD_MODNAME;
+ tty_drv->name = kstrdup(name, GFP_KERNEL);
+ if (!tty_drv->name) {
+ err = -ENOMEM;
+ goto err_tty_driver_kref_put;
+ }
+ tty_drv->type = TTY_DRIVER_TYPE_SERIAL;
+ tty_drv->subtype = SERIAL_TYPE_NORMAL;
+ tty_drv->init_termios = tty_std_termios;
+ tty_set_operations(tty_drv, &serial_ops);
+
+ /* register the tty driver */
+ err = tty_register_driver(tty_drv);
+ if (err) {
+ dev_err(dev, "tty_register_driver failed\n");
+ goto err_kfree_tty_name;
+ }
+
+ for (i = 0; i < BCM_VK_NUM_TTY; i++) {
+ struct device *tty_dev;
+
+ tty_port_init(&vk->tty[i].port);
+ tty_dev = tty_port_register_device_attr(&vk->tty[i].port,
+ tty_drv, i, dev, vk,
+ NULL);
+ if (IS_ERR(tty_dev)) {
+ err = PTR_ERR(tty_dev);
+ goto unwind;
+ }
+ vk->tty[i].is_opened = false;
+ }
+
+ INIT_WORK(&vk->tty_wq_work, bcm_vk_tty_wq_handler);
+ vk->tty_wq_thread = create_singlethread_workqueue("tty");
+ if (!vk->tty_wq_thread) {
+ dev_err(dev, "Fail to create tty workqueue thread\n");
+ err = -ENOMEM;
+ goto unwind;
+ }
+ return 0;
+
+unwind:
+ while (--i >= 0)
+ tty_port_unregister_device(&vk->tty[i].port, tty_drv, i);
+ tty_unregister_driver(tty_drv);
+
+err_kfree_tty_name:
+ kfree(tty_drv->name);
+ tty_drv->name = NULL;
+
+err_tty_driver_kref_put:
+ tty_driver_kref_put(tty_drv);
+
+ return err;
+}
+
+void bcm_vk_tty_exit(struct bcm_vk *vk)
+{
+ int i;
+
+ del_timer_sync(&vk->serial_timer);
+ for (i = 0; i < BCM_VK_NUM_TTY; ++i) {
+ tty_port_unregister_device(&vk->tty[i].port,
+ vk->tty_drv,
+ i);
+ tty_port_destroy(&vk->tty[i].port);
+ }
+ tty_unregister_driver(vk->tty_drv);
+
+ kfree(vk->tty_drv->name);
+ vk->tty_drv->name = NULL;
+
+ tty_driver_kref_put(vk->tty_drv);
+}
+
+void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk)
+{
+ struct bcm_vk_tty *vktty;
+ int i;
+
+ for (i = 0; i < BCM_VK_NUM_TTY; ++i) {
+ vktty = &vk->tty[i];
+ if (vktty->pid)
+ kill_pid(find_vpid(vktty->pid), SIGKILL, 1);
+ }
+}
+
+void bcm_vk_tty_wq_exit(struct bcm_vk *vk)
+{
+ cancel_work_sync(&vk->tty_wq_work);
+ destroy_workqueue(vk->tty_wq_thread);
+}