summaryrefslogtreecommitdiffstats
path: root/drivers/hwtracing/ptt/hisi_ptt.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:27:49 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:27:49 +0000
commitace9429bb58fd418f0c81d4c2835699bddf6bde6 (patch)
treeb2d64bc10158fdd5497876388cd68142ca374ed3 /drivers/hwtracing/ptt/hisi_ptt.c
parentInitial commit. (diff)
downloadlinux-ace9429bb58fd418f0c81d4c2835699bddf6bde6.tar.xz
linux-ace9429bb58fd418f0c81d4c2835699bddf6bde6.zip
Adding upstream version 6.6.15.upstream/6.6.15
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'drivers/hwtracing/ptt/hisi_ptt.c')
-rw-r--r--drivers/hwtracing/ptt/hisi_ptt.c1438
1 files changed, 1438 insertions, 0 deletions
diff --git a/drivers/hwtracing/ptt/hisi_ptt.c b/drivers/hwtracing/ptt/hisi_ptt.c
new file mode 100644
index 0000000000..a991ecb751
--- /dev/null
+++ b/drivers/hwtracing/ptt/hisi_ptt.c
@@ -0,0 +1,1438 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for HiSilicon PCIe tune and trace device
+ *
+ * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
+ * Author: Yicong Yang <yangyicong@hisilicon.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/cpuhotplug.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iommu.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/sysfs.h>
+#include <linux/vmalloc.h>
+
+#include "hisi_ptt.h"
+
+/* Dynamic CPU hotplug state used by PTT */
+static enum cpuhp_state hisi_ptt_pmu_online;
+
+static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt *hisi_ptt)
+{
+ u32 val;
+
+ return !readl_poll_timeout(hisi_ptt->iobase + HISI_PTT_TUNING_INT_STAT,
+ val, !(val & HISI_PTT_TUNING_INT_STAT_MASK),
+ HISI_PTT_WAIT_POLL_INTERVAL_US,
+ HISI_PTT_WAIT_TUNE_TIMEOUT_US);
+}
+
+static ssize_t hisi_ptt_tune_attr_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
+ struct dev_ext_attribute *ext_attr;
+ struct hisi_ptt_tune_desc *desc;
+ u32 reg;
+ u16 val;
+
+ ext_attr = container_of(attr, struct dev_ext_attribute, attr);
+ desc = ext_attr->var;
+
+ mutex_lock(&hisi_ptt->tune_lock);
+
+ reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
+ reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB);
+ reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB,
+ desc->event_code);
+ writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
+
+ /* Write all 1 to indicates it's the read process */
+ writel(~0U, hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
+
+ if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) {
+ mutex_unlock(&hisi_ptt->tune_lock);
+ return -ETIMEDOUT;
+ }
+
+ reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
+ reg &= HISI_PTT_TUNING_DATA_VAL_MASK;
+ val = FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK, reg);
+
+ mutex_unlock(&hisi_ptt->tune_lock);
+ return sysfs_emit(buf, "%u\n", val);
+}
+
+static ssize_t hisi_ptt_tune_attr_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
+ struct dev_ext_attribute *ext_attr;
+ struct hisi_ptt_tune_desc *desc;
+ u32 reg;
+ u16 val;
+
+ ext_attr = container_of(attr, struct dev_ext_attribute, attr);
+ desc = ext_attr->var;
+
+ if (kstrtou16(buf, 10, &val))
+ return -EINVAL;
+
+ mutex_lock(&hisi_ptt->tune_lock);
+
+ reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
+ reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB);
+ reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB,
+ desc->event_code);
+ writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
+ writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK, val),
+ hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
+
+ if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) {
+ mutex_unlock(&hisi_ptt->tune_lock);
+ return -ETIMEDOUT;
+ }
+
+ mutex_unlock(&hisi_ptt->tune_lock);
+ return count;
+}
+
+#define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \
+ static struct hisi_ptt_tune_desc _name##_desc = { \
+ .name = #_name, \
+ .event_code = (_val), \
+ }; \
+ static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \
+ .attr = __ATTR(_name, 0600, _show, _store), \
+ .var = &_name##_desc, \
+ }
+
+#define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \
+ HISI_PTT_TUNE_ATTR(_name, _val, \
+ hisi_ptt_tune_attr_show, \
+ hisi_ptt_tune_attr_store)
+
+/*
+ * The value of the tuning event are composed of two parts: main event code
+ * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is
+ * a subevent of 'Tx path QoS control' which for tuning the weight of Tx
+ * completion TLPs. See hisi_ptt.rst documentation for more information.
+ */
+#define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16))
+#define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16))
+#define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16))
+#define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16))
+#define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16))
+
+HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl, HISI_PTT_TUNE_QOS_TX_CPL);
+HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np, HISI_PTT_TUNE_QOS_TX_NP);
+HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p, HISI_PTT_TUNE_QOS_TX_P);
+HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL);
+HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL);
+
+static struct attribute *hisi_ptt_tune_attrs[] = {
+ &hisi_ptt_qos_tx_cpl_attr.attr.attr,
+ &hisi_ptt_qos_tx_np_attr.attr.attr,
+ &hisi_ptt_qos_tx_p_attr.attr.attr,
+ &hisi_ptt_rx_alloc_buf_level_attr.attr.attr,
+ &hisi_ptt_tx_alloc_buf_level_attr.attr.attr,
+ NULL,
+};
+
+static struct attribute_group hisi_ptt_tune_group = {
+ .name = "tune",
+ .attrs = hisi_ptt_tune_attrs,
+};
+
+static u16 hisi_ptt_get_filter_val(u16 devid, bool is_port)
+{
+ if (is_port)
+ return BIT(HISI_PCIE_CORE_PORT_ID(devid & 0xff));
+
+ return devid;
+}
+
+static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt)
+{
+ u32 val;
+
+ return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_STS,
+ val, val & HISI_PTT_TRACE_IDLE,
+ HISI_PTT_WAIT_POLL_INTERVAL_US,
+ HISI_PTT_WAIT_TRACE_TIMEOUT_US);
+}
+
+static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt)
+{
+ u32 val;
+
+ readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS,
+ val, !val, HISI_PTT_RESET_POLL_INTERVAL_US,
+ HISI_PTT_RESET_TIMEOUT_US);
+}
+
+static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt)
+{
+ writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
+ hisi_ptt->trace_ctrl.started = false;
+}
+
+static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt)
+{
+ struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
+ u32 val;
+ int i;
+
+ /* Check device idle before start trace */
+ if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) {
+ pci_err(hisi_ptt->pdev, "Failed to start trace, the device is still busy\n");
+ return -EBUSY;
+ }
+
+ ctrl->started = true;
+
+ /* Reset the DMA before start tracing */
+ val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
+ val |= HISI_PTT_TRACE_CTRL_RST;
+ writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
+
+ hisi_ptt_wait_dma_reset_done(hisi_ptt);
+
+ val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
+ val &= ~HISI_PTT_TRACE_CTRL_RST;
+ writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
+
+ /* Reset the index of current buffer */
+ hisi_ptt->trace_ctrl.buf_index = 0;
+
+ /* Zero the trace buffers */
+ for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++)
+ memset(ctrl->trace_buf[i].addr, 0, HISI_PTT_TRACE_BUF_SIZE);
+
+ /* Clear the interrupt status */
+ writel(HISI_PTT_TRACE_INT_STAT_MASK, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
+ writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK);
+
+ /* Set the trace control register */
+ val = FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL, ctrl->type);
+ val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL, ctrl->direction);
+ val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT, ctrl->format);
+ val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL, hisi_ptt->trace_ctrl.filter);
+ if (!hisi_ptt->trace_ctrl.is_port)
+ val |= HISI_PTT_TRACE_CTRL_FILTER_MODE;
+
+ /* Start the Trace */
+ val |= HISI_PTT_TRACE_CTRL_EN;
+ writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
+
+ return 0;
+}
+
+static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop)
+{
+ struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
+ struct perf_output_handle *handle = &ctrl->handle;
+ struct perf_event *event = handle->event;
+ struct hisi_ptt_pmu_buf *buf;
+ size_t size;
+ void *addr;
+
+ buf = perf_get_aux(handle);
+ if (!buf || !handle->size)
+ return -EINVAL;
+
+ addr = ctrl->trace_buf[ctrl->buf_index].addr;
+
+ /*
+ * If we're going to stop, read the size of already traced data from
+ * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt,
+ * the data size is always HISI_PTT_TRACE_BUF_SIZE.
+ */
+ if (stop) {
+ u32 reg;
+
+ reg = readl(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS);
+ size = FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE, reg);
+ } else {
+ size = HISI_PTT_TRACE_BUF_SIZE;
+ }
+
+ memcpy(buf->base + buf->pos, addr, size);
+ buf->pos += size;
+
+ /*
+ * Just commit the traced data if we're going to stop. Otherwise if the
+ * resident AUX buffer cannot contain the data of next trace buffer,
+ * apply a new one.
+ */
+ if (stop) {
+ perf_aux_output_end(handle, buf->pos);
+ } else if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) {
+ perf_aux_output_end(handle, buf->pos);
+
+ buf = perf_aux_output_begin(handle, event);
+ if (!buf)
+ return -EINVAL;
+
+ buf->pos = handle->head % buf->length;
+ if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) {
+ perf_aux_output_end(handle, 0);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static irqreturn_t hisi_ptt_isr(int irq, void *context)
+{
+ struct hisi_ptt *hisi_ptt = context;
+ u32 status, buf_idx;
+
+ status = readl(hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
+ if (!(status & HISI_PTT_TRACE_INT_STAT_MASK))
+ return IRQ_NONE;
+
+ buf_idx = ffs(status) - 1;
+
+ /* Clear the interrupt status of buffer @buf_idx */
+ writel(status, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
+
+ /*
+ * Update the AUX buffer and cache the current buffer index,
+ * as we need to know this and save the data when the trace
+ * is ended out of the interrupt handler. End the trace
+ * if the updating fails.
+ */
+ if (hisi_ptt_update_aux(hisi_ptt, buf_idx, false))
+ hisi_ptt_trace_end(hisi_ptt);
+ else
+ hisi_ptt->trace_ctrl.buf_index = (buf_idx + 1) % HISI_PTT_TRACE_BUF_CNT;
+
+ return IRQ_HANDLED;
+}
+
+static void hisi_ptt_irq_free_vectors(void *pdev)
+{
+ pci_free_irq_vectors(pdev);
+}
+
+static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt)
+{
+ struct pci_dev *pdev = hisi_ptt->pdev;
+ int ret;
+
+ ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
+ if (ret < 0) {
+ pci_err(pdev, "failed to allocate irq vector, ret = %d\n", ret);
+ return ret;
+ }
+
+ ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_irq_free_vectors, pdev);
+ if (ret < 0)
+ return ret;
+
+ hisi_ptt->trace_irq = pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ);
+ ret = devm_request_irq(&pdev->dev, hisi_ptt->trace_irq, hisi_ptt_isr,
+ IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME,
+ hisi_ptt);
+ if (ret) {
+ pci_err(pdev, "failed to request irq %d, ret = %d\n",
+ hisi_ptt->trace_irq, ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void hisi_ptt_del_free_filter(struct hisi_ptt *hisi_ptt,
+ struct hisi_ptt_filter_desc *filter)
+{
+ if (filter->is_port)
+ hisi_ptt->port_mask &= ~hisi_ptt_get_filter_val(filter->devid, true);
+
+ list_del(&filter->list);
+ kfree(filter->name);
+ kfree(filter);
+}
+
+static struct hisi_ptt_filter_desc *
+hisi_ptt_alloc_add_filter(struct hisi_ptt *hisi_ptt, u16 devid, bool is_port)
+{
+ struct hisi_ptt_filter_desc *filter;
+ u8 devfn = devid & 0xff;
+ char *filter_name;
+
+ filter_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d", pci_domain_nr(hisi_ptt->pdev->bus),
+ PCI_BUS_NUM(devid), PCI_SLOT(devfn), PCI_FUNC(devfn));
+ if (!filter_name) {
+ pci_err(hisi_ptt->pdev, "failed to allocate name for filter %04x:%02x:%02x.%d\n",
+ pci_domain_nr(hisi_ptt->pdev->bus), PCI_BUS_NUM(devid),
+ PCI_SLOT(devfn), PCI_FUNC(devfn));
+ return NULL;
+ }
+
+ filter = kzalloc(sizeof(*filter), GFP_KERNEL);
+ if (!filter) {
+ pci_err(hisi_ptt->pdev, "failed to add filter for %s\n",
+ filter_name);
+ kfree(filter_name);
+ return NULL;
+ }
+
+ filter->name = filter_name;
+ filter->is_port = is_port;
+ filter->devid = devid;
+
+ if (filter->is_port) {
+ list_add_tail(&filter->list, &hisi_ptt->port_filters);
+
+ /* Update the available port mask */
+ hisi_ptt->port_mask |= hisi_ptt_get_filter_val(filter->devid, true);
+ } else {
+ list_add_tail(&filter->list, &hisi_ptt->req_filters);
+ }
+
+ return filter;
+}
+
+static ssize_t hisi_ptt_filter_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct hisi_ptt_filter_desc *filter;
+ unsigned long filter_val;
+
+ filter = container_of(attr, struct hisi_ptt_filter_desc, attr);
+ filter_val = hisi_ptt_get_filter_val(filter->devid, filter->is_port) |
+ (filter->is_port ? HISI_PTT_PMU_FILTER_IS_PORT : 0);
+
+ return sysfs_emit(buf, "0x%05lx\n", filter_val);
+}
+
+static int hisi_ptt_create_rp_filter_attr(struct hisi_ptt *hisi_ptt,
+ struct hisi_ptt_filter_desc *filter)
+{
+ struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj;
+
+ sysfs_attr_init(&filter->attr.attr);
+ filter->attr.attr.name = filter->name;
+ filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */
+ filter->attr.show = hisi_ptt_filter_show;
+
+ return sysfs_add_file_to_group(kobj, &filter->attr.attr,
+ HISI_PTT_RP_FILTERS_GRP_NAME);
+}
+
+static void hisi_ptt_remove_rp_filter_attr(struct hisi_ptt *hisi_ptt,
+ struct hisi_ptt_filter_desc *filter)
+{
+ struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj;
+
+ sysfs_remove_file_from_group(kobj, &filter->attr.attr,
+ HISI_PTT_RP_FILTERS_GRP_NAME);
+}
+
+static int hisi_ptt_create_req_filter_attr(struct hisi_ptt *hisi_ptt,
+ struct hisi_ptt_filter_desc *filter)
+{
+ struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj;
+
+ sysfs_attr_init(&filter->attr.attr);
+ filter->attr.attr.name = filter->name;
+ filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */
+ filter->attr.show = hisi_ptt_filter_show;
+
+ return sysfs_add_file_to_group(kobj, &filter->attr.attr,
+ HISI_PTT_REQ_FILTERS_GRP_NAME);
+}
+
+static void hisi_ptt_remove_req_filter_attr(struct hisi_ptt *hisi_ptt,
+ struct hisi_ptt_filter_desc *filter)
+{
+ struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj;
+
+ sysfs_remove_file_from_group(kobj, &filter->attr.attr,
+ HISI_PTT_REQ_FILTERS_GRP_NAME);
+}
+
+static int hisi_ptt_create_filter_attr(struct hisi_ptt *hisi_ptt,
+ struct hisi_ptt_filter_desc *filter)
+{
+ int ret;
+
+ if (filter->is_port)
+ ret = hisi_ptt_create_rp_filter_attr(hisi_ptt, filter);
+ else
+ ret = hisi_ptt_create_req_filter_attr(hisi_ptt, filter);
+
+ if (ret)
+ pci_err(hisi_ptt->pdev, "failed to create sysfs attribute for filter %s\n",
+ filter->name);
+
+ return ret;
+}
+
+static void hisi_ptt_remove_filter_attr(struct hisi_ptt *hisi_ptt,
+ struct hisi_ptt_filter_desc *filter)
+{
+ if (filter->is_port)
+ hisi_ptt_remove_rp_filter_attr(hisi_ptt, filter);
+ else
+ hisi_ptt_remove_req_filter_attr(hisi_ptt, filter);
+}
+
+static void hisi_ptt_remove_all_filter_attributes(void *data)
+{
+ struct hisi_ptt_filter_desc *filter;
+ struct hisi_ptt *hisi_ptt = data;
+
+ mutex_lock(&hisi_ptt->filter_lock);
+
+ list_for_each_entry(filter, &hisi_ptt->req_filters, list)
+ hisi_ptt_remove_filter_attr(hisi_ptt, filter);
+
+ list_for_each_entry(filter, &hisi_ptt->port_filters, list)
+ hisi_ptt_remove_filter_attr(hisi_ptt, filter);
+
+ hisi_ptt->sysfs_inited = false;
+ mutex_unlock(&hisi_ptt->filter_lock);
+}
+
+static int hisi_ptt_init_filter_attributes(struct hisi_ptt *hisi_ptt)
+{
+ struct hisi_ptt_filter_desc *filter;
+ int ret;
+
+ mutex_lock(&hisi_ptt->filter_lock);
+
+ /*
+ * Register the reset callback in the first stage. In reset we traverse
+ * the filters list to remove the sysfs attributes so the callback can
+ * be called safely even without below filter attributes creation.
+ */
+ ret = devm_add_action(&hisi_ptt->pdev->dev,
+ hisi_ptt_remove_all_filter_attributes,
+ hisi_ptt);
+ if (ret)
+ goto out;
+
+ list_for_each_entry(filter, &hisi_ptt->port_filters, list) {
+ ret = hisi_ptt_create_filter_attr(hisi_ptt, filter);
+ if (ret)
+ goto out;
+ }
+
+ list_for_each_entry(filter, &hisi_ptt->req_filters, list) {
+ ret = hisi_ptt_create_filter_attr(hisi_ptt, filter);
+ if (ret)
+ goto out;
+ }
+
+ hisi_ptt->sysfs_inited = true;
+out:
+ mutex_unlock(&hisi_ptt->filter_lock);
+ return ret;
+}
+
+static void hisi_ptt_update_filters(struct work_struct *work)
+{
+ struct delayed_work *delayed_work = to_delayed_work(work);
+ struct hisi_ptt_filter_update_info info;
+ struct hisi_ptt_filter_desc *filter;
+ struct hisi_ptt *hisi_ptt;
+
+ hisi_ptt = container_of(delayed_work, struct hisi_ptt, work);
+
+ if (!mutex_trylock(&hisi_ptt->filter_lock)) {
+ schedule_delayed_work(&hisi_ptt->work, HISI_PTT_WORK_DELAY_MS);
+ return;
+ }
+
+ while (kfifo_get(&hisi_ptt->filter_update_kfifo, &info)) {
+ if (info.is_add) {
+ /*
+ * Notify the users if failed to add this filter, others
+ * still work and available. See the comments in
+ * hisi_ptt_init_filters().
+ */
+ filter = hisi_ptt_alloc_add_filter(hisi_ptt, info.devid, info.is_port);
+ if (!filter)
+ continue;
+
+ /*
+ * If filters' sysfs entries hasn't been initialized,
+ * then we're still at probe stage. Add the filters to
+ * the list and later hisi_ptt_init_filter_attributes()
+ * will create sysfs attributes for all the filters.
+ */
+ if (hisi_ptt->sysfs_inited &&
+ hisi_ptt_create_filter_attr(hisi_ptt, filter)) {
+ hisi_ptt_del_free_filter(hisi_ptt, filter);
+ continue;
+ }
+ } else {
+ struct hisi_ptt_filter_desc *tmp;
+ struct list_head *target_list;
+
+ target_list = info.is_port ? &hisi_ptt->port_filters :
+ &hisi_ptt->req_filters;
+
+ list_for_each_entry_safe(filter, tmp, target_list, list)
+ if (filter->devid == info.devid) {
+ if (hisi_ptt->sysfs_inited)
+ hisi_ptt_remove_filter_attr(hisi_ptt, filter);
+
+ hisi_ptt_del_free_filter(hisi_ptt, filter);
+ break;
+ }
+ }
+ }
+
+ mutex_unlock(&hisi_ptt->filter_lock);
+}
+
+/*
+ * A PCI bus notifier is used here for dynamically updating the filter
+ * list.
+ */
+static int hisi_ptt_notifier_call(struct notifier_block *nb, unsigned long action,
+ void *data)
+{
+ struct hisi_ptt *hisi_ptt = container_of(nb, struct hisi_ptt, hisi_ptt_nb);
+ struct hisi_ptt_filter_update_info info;
+ struct pci_dev *pdev, *root_port;
+ struct device *dev = data;
+ u32 port_devid;
+
+ pdev = to_pci_dev(dev);
+ root_port = pcie_find_root_port(pdev);
+ if (!root_port)
+ return 0;
+
+ port_devid = pci_dev_id(root_port);
+ if (port_devid < hisi_ptt->lower_bdf ||
+ port_devid > hisi_ptt->upper_bdf)
+ return 0;
+
+ info.is_port = pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT;
+ info.devid = pci_dev_id(pdev);
+
+ switch (action) {
+ case BUS_NOTIFY_ADD_DEVICE:
+ info.is_add = true;
+ break;
+ case BUS_NOTIFY_DEL_DEVICE:
+ info.is_add = false;
+ break;
+ default:
+ return 0;
+ }
+
+ /*
+ * The FIFO size is 16 which is sufficient for almost all the cases,
+ * since each PCIe core will have most 8 Root Ports (typically only
+ * 1~4 Root Ports). On failure log the failed filter and let user
+ * handle it.
+ */
+ if (kfifo_in_spinlocked(&hisi_ptt->filter_update_kfifo, &info, 1,
+ &hisi_ptt->filter_update_lock))
+ schedule_delayed_work(&hisi_ptt->work, 0);
+ else
+ pci_warn(hisi_ptt->pdev,
+ "filter update fifo overflow for target %s\n",
+ pci_name(pdev));
+
+ return 0;
+}
+
+static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data)
+{
+ struct pci_dev *root_port = pcie_find_root_port(pdev);
+ struct hisi_ptt_filter_desc *filter;
+ struct hisi_ptt *hisi_ptt = data;
+ u32 port_devid;
+
+ if (!root_port)
+ return 0;
+
+ port_devid = pci_dev_id(root_port);
+ if (port_devid < hisi_ptt->lower_bdf ||
+ port_devid > hisi_ptt->upper_bdf)
+ return 0;
+
+ /*
+ * We won't fail the probe if filter allocation failed here. The filters
+ * should be partial initialized and users would know which filter fails
+ * through the log. Other functions of PTT device are still available.
+ */
+ filter = hisi_ptt_alloc_add_filter(hisi_ptt, pci_dev_id(pdev),
+ pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT);
+ if (!filter)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void hisi_ptt_release_filters(void *data)
+{
+ struct hisi_ptt_filter_desc *filter, *tmp;
+ struct hisi_ptt *hisi_ptt = data;
+
+ list_for_each_entry_safe(filter, tmp, &hisi_ptt->req_filters, list)
+ hisi_ptt_del_free_filter(hisi_ptt, filter);
+
+ list_for_each_entry_safe(filter, tmp, &hisi_ptt->port_filters, list)
+ hisi_ptt_del_free_filter(hisi_ptt, filter);
+}
+
+static int hisi_ptt_config_trace_buf(struct hisi_ptt *hisi_ptt)
+{
+ struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
+ struct device *dev = &hisi_ptt->pdev->dev;
+ int i;
+
+ ctrl->trace_buf = devm_kcalloc(dev, HISI_PTT_TRACE_BUF_CNT,
+ sizeof(*ctrl->trace_buf), GFP_KERNEL);
+ if (!ctrl->trace_buf)
+ return -ENOMEM;
+
+ for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; ++i) {
+ ctrl->trace_buf[i].addr = dmam_alloc_coherent(dev, HISI_PTT_TRACE_BUF_SIZE,
+ &ctrl->trace_buf[i].dma,
+ GFP_KERNEL);
+ if (!ctrl->trace_buf[i].addr)
+ return -ENOMEM;
+ }
+
+ /* Configure the trace DMA buffer */
+ for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) {
+ writel(lower_32_bits(ctrl->trace_buf[i].dma),
+ hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_LO_0 +
+ i * HISI_PTT_TRACE_ADDR_STRIDE);
+ writel(upper_32_bits(ctrl->trace_buf[i].dma),
+ hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_HI_0 +
+ i * HISI_PTT_TRACE_ADDR_STRIDE);
+ }
+ writel(HISI_PTT_TRACE_BUF_SIZE, hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_SIZE);
+
+ return 0;
+}
+
+static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt)
+{
+ struct pci_dev *pdev = hisi_ptt->pdev;
+ struct pci_bus *bus;
+ int ret;
+ u32 reg;
+
+ INIT_DELAYED_WORK(&hisi_ptt->work, hisi_ptt_update_filters);
+ INIT_KFIFO(hisi_ptt->filter_update_kfifo);
+ spin_lock_init(&hisi_ptt->filter_update_lock);
+
+ INIT_LIST_HEAD(&hisi_ptt->port_filters);
+ INIT_LIST_HEAD(&hisi_ptt->req_filters);
+ mutex_init(&hisi_ptt->filter_lock);
+
+ ret = hisi_ptt_config_trace_buf(hisi_ptt);
+ if (ret)
+ return ret;
+
+ /*
+ * The device range register provides the information about the root
+ * ports which the RCiEP can control and trace. The RCiEP and the root
+ * ports which it supports are on the same PCIe core, with same domain
+ * number but maybe different bus number. The device range register
+ * will tell us which root ports we can support, Bit[31:16] indicates
+ * the upper BDF numbers of the root port, while Bit[15:0] indicates
+ * the lower.
+ */
+ reg = readl(hisi_ptt->iobase + HISI_PTT_DEVICE_RANGE);
+ hisi_ptt->upper_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER, reg);
+ hisi_ptt->lower_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER, reg);
+
+ bus = pci_find_bus(pci_domain_nr(pdev->bus), PCI_BUS_NUM(hisi_ptt->upper_bdf));
+ if (bus)
+ pci_walk_bus(bus, hisi_ptt_init_filters, hisi_ptt);
+
+ ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_release_filters, hisi_ptt);
+ if (ret)
+ return ret;
+
+ hisi_ptt->trace_ctrl.on_cpu = -1;
+ return 0;
+}
+
+static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
+ const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev));
+
+ return cpumap_print_to_pagebuf(true, buf, cpumask);
+}
+static DEVICE_ATTR_RO(cpumask);
+
+static struct attribute *hisi_ptt_cpumask_attrs[] = {
+ &dev_attr_cpumask.attr,
+ NULL
+};
+
+static const struct attribute_group hisi_ptt_cpumask_attr_group = {
+ .attrs = hisi_ptt_cpumask_attrs,
+};
+
+/*
+ * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester
+ * filter. Bit[15:0] indicates the filter value, for Root Port filter it's
+ * a bit mask of desired ports and for Requester filter it's the Requester ID
+ * of the desired PCIe function. Bit[18:16] is reserved for extension.
+ *
+ * See hisi_ptt.rst documentation for detailed information.
+ */
+PMU_FORMAT_ATTR(filter, "config:0-19");
+PMU_FORMAT_ATTR(direction, "config:20-23");
+PMU_FORMAT_ATTR(type, "config:24-31");
+PMU_FORMAT_ATTR(format, "config:32-35");
+
+static struct attribute *hisi_ptt_pmu_format_attrs[] = {
+ &format_attr_filter.attr,
+ &format_attr_direction.attr,
+ &format_attr_type.attr,
+ &format_attr_format.attr,
+ NULL
+};
+
+static struct attribute_group hisi_ptt_pmu_format_group = {
+ .name = "format",
+ .attrs = hisi_ptt_pmu_format_attrs,
+};
+
+static ssize_t hisi_ptt_filter_multiselect_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct dev_ext_attribute *ext_attr;
+
+ ext_attr = container_of(attr, struct dev_ext_attribute, attr);
+ return sysfs_emit(buf, "%s\n", (char *)ext_attr->var);
+}
+
+static struct dev_ext_attribute root_port_filters_multiselect = {
+ .attr = {
+ .attr = { .name = "multiselect", .mode = 0400 },
+ .show = hisi_ptt_filter_multiselect_show,
+ },
+ .var = "1",
+};
+
+static struct attribute *hisi_ptt_pmu_root_ports_attrs[] = {
+ &root_port_filters_multiselect.attr.attr,
+ NULL
+};
+
+static struct attribute_group hisi_ptt_pmu_root_ports_group = {
+ .name = HISI_PTT_RP_FILTERS_GRP_NAME,
+ .attrs = hisi_ptt_pmu_root_ports_attrs,
+};
+
+static struct dev_ext_attribute requester_filters_multiselect = {
+ .attr = {
+ .attr = { .name = "multiselect", .mode = 0400 },
+ .show = hisi_ptt_filter_multiselect_show,
+ },
+ .var = "0",
+};
+
+static struct attribute *hisi_ptt_pmu_requesters_attrs[] = {
+ &requester_filters_multiselect.attr.attr,
+ NULL
+};
+
+static struct attribute_group hisi_ptt_pmu_requesters_group = {
+ .name = HISI_PTT_REQ_FILTERS_GRP_NAME,
+ .attrs = hisi_ptt_pmu_requesters_attrs,
+};
+
+static const struct attribute_group *hisi_ptt_pmu_groups[] = {
+ &hisi_ptt_cpumask_attr_group,
+ &hisi_ptt_pmu_format_group,
+ &hisi_ptt_tune_group,
+ &hisi_ptt_pmu_root_ports_group,
+ &hisi_ptt_pmu_requesters_group,
+ NULL
+};
+
+static int hisi_ptt_trace_valid_direction(u32 val)
+{
+ /*
+ * The direction values have different effects according to the data
+ * format (specified in the parentheses). TLP set A/B means different
+ * set of TLP types. See hisi_ptt.rst documentation for more details.
+ */
+ static const u32 hisi_ptt_trace_available_direction[] = {
+ 0, /* inbound(4DW) or reserved(8DW) */
+ 1, /* outbound(4DW) */
+ 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */
+ 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */
+ };
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_direction); i++) {
+ if (val == hisi_ptt_trace_available_direction[i])
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int hisi_ptt_trace_valid_type(u32 val)
+{
+ /* Different types can be set simultaneously */
+ static const u32 hisi_ptt_trace_available_type[] = {
+ 1, /* posted_request */
+ 2, /* non-posted_request */
+ 4, /* completion */
+ };
+ int i;
+
+ if (!val)
+ return -EINVAL;
+
+ /*
+ * Walk the available list and clear the valid bits of
+ * the config. If there is any resident bit after the
+ * walk then the config is invalid.
+ */
+ for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_type); i++)
+ val &= ~hisi_ptt_trace_available_type[i];
+
+ if (val)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int hisi_ptt_trace_valid_format(u32 val)
+{
+ static const u32 hisi_ptt_trace_availble_format[] = {
+ 0, /* 4DW */
+ 1, /* 8DW */
+ };
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_availble_format); i++) {
+ if (val == hisi_ptt_trace_availble_format[i])
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config)
+{
+ unsigned long val, port_mask = hisi_ptt->port_mask;
+ struct hisi_ptt_filter_desc *filter;
+ int ret = 0;
+
+ hisi_ptt->trace_ctrl.is_port = FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT, config);
+ val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, config);
+
+ /*
+ * Port filters are defined as bit mask. For port filters, check
+ * the bits in the @val are within the range of hisi_ptt->port_mask
+ * and whether it's empty or not, otherwise user has specified
+ * some unsupported root ports.
+ *
+ * For Requester ID filters, walk the available filter list to see
+ * whether we have one matched.
+ */
+ mutex_lock(&hisi_ptt->filter_lock);
+ if (!hisi_ptt->trace_ctrl.is_port) {
+ list_for_each_entry(filter, &hisi_ptt->req_filters, list) {
+ if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port))
+ goto out;
+ }
+ } else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) {
+ goto out;
+ }
+
+ ret = -EINVAL;
+out:
+ mutex_unlock(&hisi_ptt->filter_lock);
+ return ret;
+}
+
+static void hisi_ptt_pmu_init_configs(struct hisi_ptt *hisi_ptt, struct perf_event *event)
+{
+ struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
+ u32 val;
+
+ val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, event->attr.config);
+ hisi_ptt->trace_ctrl.filter = val;
+
+ val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config);
+ ctrl->direction = val;
+
+ val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config);
+ ctrl->type = val;
+
+ val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config);
+ ctrl->format = val;
+}
+
+static int hisi_ptt_pmu_event_init(struct perf_event *event)
+{
+ struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
+ int ret;
+ u32 val;
+
+ if (event->cpu < 0) {
+ dev_dbg(event->pmu->dev, "Per-task mode not supported\n");
+ return -EOPNOTSUPP;
+ }
+
+ if (event->attach_state & PERF_ATTACH_TASK)
+ return -EOPNOTSUPP;
+
+ if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type)
+ return -ENOENT;
+
+ ret = hisi_ptt_trace_valid_filter(hisi_ptt, event->attr.config);
+ if (ret < 0)
+ return ret;
+
+ val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config);
+ ret = hisi_ptt_trace_valid_direction(val);
+ if (ret < 0)
+ return ret;
+
+ val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config);
+ ret = hisi_ptt_trace_valid_type(val);
+ if (ret < 0)
+ return ret;
+
+ val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config);
+ return hisi_ptt_trace_valid_format(val);
+}
+
+static void *hisi_ptt_pmu_setup_aux(struct perf_event *event, void **pages,
+ int nr_pages, bool overwrite)
+{
+ struct hisi_ptt_pmu_buf *buf;
+ struct page **pagelist;
+ int i;
+
+ if (overwrite) {
+ dev_warn(event->pmu->dev, "Overwrite mode is not supported\n");
+ return NULL;
+ }
+
+ /* If the pages size less than buffers, we cannot start trace */
+ if (nr_pages < HISI_PTT_TRACE_TOTAL_BUF_SIZE / PAGE_SIZE)
+ return NULL;
+
+ buf = kzalloc(sizeof(*buf), GFP_KERNEL);
+ if (!buf)
+ return NULL;
+
+ pagelist = kcalloc(nr_pages, sizeof(*pagelist), GFP_KERNEL);
+ if (!pagelist)
+ goto err;
+
+ for (i = 0; i < nr_pages; i++)
+ pagelist[i] = virt_to_page(pages[i]);
+
+ buf->base = vmap(pagelist, nr_pages, VM_MAP, PAGE_KERNEL);
+ if (!buf->base) {
+ kfree(pagelist);
+ goto err;
+ }
+
+ buf->nr_pages = nr_pages;
+ buf->length = nr_pages * PAGE_SIZE;
+ buf->pos = 0;
+
+ kfree(pagelist);
+ return buf;
+err:
+ kfree(buf);
+ return NULL;
+}
+
+static void hisi_ptt_pmu_free_aux(void *aux)
+{
+ struct hisi_ptt_pmu_buf *buf = aux;
+
+ vunmap(buf->base);
+ kfree(buf);
+}
+
+static void hisi_ptt_pmu_start(struct perf_event *event, int flags)
+{
+ struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
+ struct perf_output_handle *handle = &hisi_ptt->trace_ctrl.handle;
+ struct hw_perf_event *hwc = &event->hw;
+ struct device *dev = event->pmu->dev;
+ struct hisi_ptt_pmu_buf *buf;
+ int cpu = event->cpu;
+ int ret;
+
+ hwc->state = 0;
+
+ /* Serialize the perf process if user specified several CPUs */
+ spin_lock(&hisi_ptt->pmu_lock);
+ if (hisi_ptt->trace_ctrl.started) {
+ dev_dbg(dev, "trace has already started\n");
+ goto stop;
+ }
+
+ /*
+ * Handle the interrupt on the same cpu which starts the trace to avoid
+ * context mismatch. Otherwise we'll trigger the WARN from the perf
+ * core in event_function_local(). If CPU passed is offline we'll fail
+ * here, just log it since we can do nothing here.
+ */
+ ret = irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(cpu));
+ if (ret)
+ dev_warn(dev, "failed to set the affinity of trace interrupt\n");
+
+ hisi_ptt->trace_ctrl.on_cpu = cpu;
+
+ buf = perf_aux_output_begin(handle, event);
+ if (!buf) {
+ dev_dbg(dev, "aux output begin failed\n");
+ goto stop;
+ }
+
+ buf->pos = handle->head % buf->length;
+
+ hisi_ptt_pmu_init_configs(hisi_ptt, event);
+
+ ret = hisi_ptt_trace_start(hisi_ptt);
+ if (ret) {
+ dev_dbg(dev, "trace start failed, ret = %d\n", ret);
+ perf_aux_output_end(handle, 0);
+ goto stop;
+ }
+
+ spin_unlock(&hisi_ptt->pmu_lock);
+ return;
+stop:
+ event->hw.state |= PERF_HES_STOPPED;
+ spin_unlock(&hisi_ptt->pmu_lock);
+}
+
+static void hisi_ptt_pmu_stop(struct perf_event *event, int flags)
+{
+ struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+
+ if (hwc->state & PERF_HES_STOPPED)
+ return;
+
+ spin_lock(&hisi_ptt->pmu_lock);
+ if (hisi_ptt->trace_ctrl.started) {
+ hisi_ptt_trace_end(hisi_ptt);
+
+ if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt))
+ dev_warn(event->pmu->dev, "Device is still busy\n");
+
+ hisi_ptt_update_aux(hisi_ptt, hisi_ptt->trace_ctrl.buf_index, true);
+ }
+ spin_unlock(&hisi_ptt->pmu_lock);
+
+ hwc->state |= PERF_HES_STOPPED;
+ perf_event_update_userpage(event);
+ hwc->state |= PERF_HES_UPTODATE;
+}
+
+static int hisi_ptt_pmu_add(struct perf_event *event, int flags)
+{
+ struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
+ struct hw_perf_event *hwc = &event->hw;
+ int cpu = event->cpu;
+
+ /* Only allow the cpus on the device's node to add the event */
+ if (!cpumask_test_cpu(cpu, cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev))))
+ return 0;
+
+ hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+
+ if (flags & PERF_EF_START) {
+ hisi_ptt_pmu_start(event, PERF_EF_RELOAD);
+ if (hwc->state & PERF_HES_STOPPED)
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void hisi_ptt_pmu_del(struct perf_event *event, int flags)
+{
+ hisi_ptt_pmu_stop(event, PERF_EF_UPDATE);
+}
+
+static void hisi_ptt_pmu_read(struct perf_event *event)
+{
+}
+
+static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node)
+{
+ cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node);
+}
+
+static void hisi_ptt_unregister_pmu(void *pmu)
+{
+ perf_pmu_unregister(pmu);
+}
+
+static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt)
+{
+ u16 core_id, sicl_id;
+ char *pmu_name;
+ u32 reg;
+ int ret;
+
+ ret = cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online,
+ &hisi_ptt->hotplug_node);
+ if (ret)
+ return ret;
+
+ ret = devm_add_action_or_reset(&hisi_ptt->pdev->dev,
+ hisi_ptt_remove_cpuhp_instance,
+ &hisi_ptt->hotplug_node);
+ if (ret)
+ return ret;
+
+ mutex_init(&hisi_ptt->tune_lock);
+ spin_lock_init(&hisi_ptt->pmu_lock);
+
+ hisi_ptt->hisi_ptt_pmu = (struct pmu) {
+ .module = THIS_MODULE,
+ .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_NO_EXCLUDE,
+ .task_ctx_nr = perf_sw_context,
+ .attr_groups = hisi_ptt_pmu_groups,
+ .event_init = hisi_ptt_pmu_event_init,
+ .setup_aux = hisi_ptt_pmu_setup_aux,
+ .free_aux = hisi_ptt_pmu_free_aux,
+ .start = hisi_ptt_pmu_start,
+ .stop = hisi_ptt_pmu_stop,
+ .add = hisi_ptt_pmu_add,
+ .del = hisi_ptt_pmu_del,
+ .read = hisi_ptt_pmu_read,
+ };
+
+ reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION);
+ core_id = FIELD_GET(HISI_PTT_CORE_ID, reg);
+ sicl_id = FIELD_GET(HISI_PTT_SICL_ID, reg);
+
+ pmu_name = devm_kasprintf(&hisi_ptt->pdev->dev, GFP_KERNEL, "hisi_ptt%u_%u",
+ sicl_id, core_id);
+ if (!pmu_name)
+ return -ENOMEM;
+
+ ret = perf_pmu_register(&hisi_ptt->hisi_ptt_pmu, pmu_name, -1);
+ if (ret)
+ return ret;
+
+ return devm_add_action_or_reset(&hisi_ptt->pdev->dev,
+ hisi_ptt_unregister_pmu,
+ &hisi_ptt->hisi_ptt_pmu);
+}
+
+static void hisi_ptt_unregister_filter_update_notifier(void *data)
+{
+ struct hisi_ptt *hisi_ptt = data;
+
+ bus_unregister_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb);
+
+ /* Cancel any work that has been queued */
+ cancel_delayed_work_sync(&hisi_ptt->work);
+}
+
+/* Register the bus notifier for dynamically updating the filter list */
+static int hisi_ptt_register_filter_update_notifier(struct hisi_ptt *hisi_ptt)
+{
+ int ret;
+
+ hisi_ptt->hisi_ptt_nb.notifier_call = hisi_ptt_notifier_call;
+ ret = bus_register_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb);
+ if (ret)
+ return ret;
+
+ return devm_add_action_or_reset(&hisi_ptt->pdev->dev,
+ hisi_ptt_unregister_filter_update_notifier,
+ hisi_ptt);
+}
+
+/*
+ * The DMA of PTT trace can only use direct mappings due to some
+ * hardware restriction. Check whether there is no IOMMU or the
+ * policy of the IOMMU domain is passthrough, otherwise the trace
+ * cannot work.
+ *
+ * The PTT device is supposed to behind an ARM SMMUv3, which
+ * should have passthrough the device by a quirk.
+ */
+static int hisi_ptt_check_iommu_mapping(struct pci_dev *pdev)
+{
+ struct iommu_domain *iommu_domain;
+
+ iommu_domain = iommu_get_domain_for_dev(&pdev->dev);
+ if (!iommu_domain || iommu_domain->type == IOMMU_DOMAIN_IDENTITY)
+ return 0;
+
+ return -EOPNOTSUPP;
+}
+
+static int hisi_ptt_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+{
+ struct hisi_ptt *hisi_ptt;
+ int ret;
+
+ ret = hisi_ptt_check_iommu_mapping(pdev);
+ if (ret) {
+ pci_err(pdev, "requires direct DMA mappings\n");
+ return ret;
+ }
+
+ hisi_ptt = devm_kzalloc(&pdev->dev, sizeof(*hisi_ptt), GFP_KERNEL);
+ if (!hisi_ptt)
+ return -ENOMEM;
+
+ hisi_ptt->pdev = pdev;
+ pci_set_drvdata(pdev, hisi_ptt);
+
+ ret = pcim_enable_device(pdev);
+ if (ret) {
+ pci_err(pdev, "failed to enable device, ret = %d\n", ret);
+ return ret;
+ }
+
+ ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME);
+ if (ret) {
+ pci_err(pdev, "failed to remap io memory, ret = %d\n", ret);
+ return ret;
+ }
+
+ hisi_ptt->iobase = pcim_iomap_table(pdev)[2];
+
+ ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
+ if (ret) {
+ pci_err(pdev, "failed to set 64 bit dma mask, ret = %d\n", ret);
+ return ret;
+ }
+
+ pci_set_master(pdev);
+
+ ret = hisi_ptt_register_irq(hisi_ptt);
+ if (ret)
+ return ret;
+
+ ret = hisi_ptt_init_ctrls(hisi_ptt);
+ if (ret) {
+ pci_err(pdev, "failed to init controls, ret = %d\n", ret);
+ return ret;
+ }
+
+ ret = hisi_ptt_register_filter_update_notifier(hisi_ptt);
+ if (ret)
+ pci_warn(pdev, "failed to register filter update notifier, ret = %d", ret);
+
+ ret = hisi_ptt_register_pmu(hisi_ptt);
+ if (ret) {
+ pci_err(pdev, "failed to register PMU device, ret = %d", ret);
+ return ret;
+ }
+
+ ret = hisi_ptt_init_filter_attributes(hisi_ptt);
+ if (ret) {
+ pci_err(pdev, "failed to init sysfs filter attributes, ret = %d", ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static const struct pci_device_id hisi_ptt_id_tbl[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12e) },
+ { }
+};
+MODULE_DEVICE_TABLE(pci, hisi_ptt_id_tbl);
+
+static struct pci_driver hisi_ptt_driver = {
+ .name = DRV_NAME,
+ .id_table = hisi_ptt_id_tbl,
+ .probe = hisi_ptt_probe,
+};
+
+static int hisi_ptt_cpu_teardown(unsigned int cpu, struct hlist_node *node)
+{
+ struct hisi_ptt *hisi_ptt;
+ struct device *dev;
+ int target, src;
+
+ hisi_ptt = hlist_entry_safe(node, struct hisi_ptt, hotplug_node);
+ src = hisi_ptt->trace_ctrl.on_cpu;
+ dev = hisi_ptt->hisi_ptt_pmu.dev;
+
+ if (!hisi_ptt->trace_ctrl.started || src != cpu)
+ return 0;
+
+ target = cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)), cpu);
+ if (target >= nr_cpu_ids) {
+ dev_err(dev, "no available cpu for perf context migration\n");
+ return 0;
+ }
+
+ perf_pmu_migrate_context(&hisi_ptt->hisi_ptt_pmu, src, target);
+
+ /*
+ * Also make sure the interrupt bind to the migrated CPU as well. Warn
+ * the user on failure here.
+ */
+ if (irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(target)))
+ dev_warn(dev, "failed to set the affinity of trace interrupt\n");
+
+ hisi_ptt->trace_ctrl.on_cpu = target;
+ return 0;
+}
+
+static int __init hisi_ptt_init(void)
+{
+ int ret;
+
+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRV_NAME, NULL,
+ hisi_ptt_cpu_teardown);
+ if (ret < 0)
+ return ret;
+ hisi_ptt_pmu_online = ret;
+
+ ret = pci_register_driver(&hisi_ptt_driver);
+ if (ret)
+ cpuhp_remove_multi_state(hisi_ptt_pmu_online);
+
+ return ret;
+}
+module_init(hisi_ptt_init);
+
+static void __exit hisi_ptt_exit(void)
+{
+ pci_unregister_driver(&hisi_ptt_driver);
+ cpuhp_remove_multi_state(hisi_ptt_pmu_online);
+}
+module_exit(hisi_ptt_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Yicong Yang <yangyicong@hisilicon.com>");
+MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device");