diff options
Diffstat (limited to '')
| -rw-r--r-- | drivers/rpmsg/Kconfig | 84 | ||||
| -rw-r--r-- | drivers/rpmsg/Makefile | 12 | ||||
| -rw-r--r-- | drivers/rpmsg/mtk_rpmsg.c | 410 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_native.c | 1882 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_native.h | 41 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_rpm.c | 405 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_smem.c | 378 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_ssr.c | 167 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_smd.c | 1620 | ||||
| -rw-r--r-- | drivers/rpmsg/rpmsg_char.c | 569 | ||||
| -rw-r--r-- | drivers/rpmsg/rpmsg_char.h | 46 | ||||
| -rw-r--r-- | drivers/rpmsg/rpmsg_core.c | 741 | ||||
| -rw-r--r-- | drivers/rpmsg/rpmsg_ctrl.c | 245 | ||||
| -rw-r--r-- | drivers/rpmsg/rpmsg_internal.h | 102 | ||||
| -rw-r--r-- | drivers/rpmsg/rpmsg_ns.c | 124 | ||||
| -rw-r--r-- | drivers/rpmsg/virtio_rpmsg_bus.c | 1082 |
16 files changed, 7908 insertions, 0 deletions
diff --git a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig new file mode 100644 index 0000000000..d3795860f5 --- /dev/null +++ b/drivers/rpmsg/Kconfig @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Rpmsg drivers" + +# RPMSG always gets selected by whoever wants it +config RPMSG + tristate + +config RPMSG_CHAR + tristate "RPMSG device interface" + depends on RPMSG + depends on NET + help + Say Y here to export rpmsg endpoints as device files, usually found + in /dev. They make it possible for user-space programs to send and + receive rpmsg packets. + +config RPMSG_CTRL + tristate "RPMSG control interface" + depends on RPMSG && ( RPMSG_CHAR || RPMSG_CHAR=n ) + help + Say Y here to enable the support of the /dev/rpmsg_ctrlX API. This API + allows user-space programs to create endpoints with specific service name, + source and destination addresses. + +config RPMSG_NS + tristate "RPMSG name service announcement" + depends on RPMSG + help + Say Y here to enable the support of the name service announcement + channel that probes the associated RPMsg device on remote endpoint + service announcement. + +config RPMSG_MTK_SCP + tristate "MediaTek SCP" + depends on MTK_SCP + select RPMSG + help + Say y here to enable support providing communication channels to + remote processors in MediaTek platforms. + This use IPI and IPC to communicate with remote processors. + +config RPMSG_QCOM_GLINK + tristate + select RPMSG + +config RPMSG_QCOM_GLINK_RPM + tristate "Qualcomm RPM Glink driver" + select RPMSG_QCOM_GLINK + depends on HAS_IOMEM + depends on MAILBOX + help + Say y here to enable support for the GLINK RPM communication driver, + which serves as a channel for communication with the RPM in GLINK + enabled systems. + +config RPMSG_QCOM_GLINK_SMEM + tristate "Qualcomm SMEM Glink driver" + select RPMSG_QCOM_GLINK + depends on MAILBOX + depends on QCOM_SMEM + help + Say y here to enable support for the GLINK SMEM communication driver, + which provides support for using the GLINK communication protocol + over SMEM. + +config RPMSG_QCOM_SMD + tristate "Qualcomm Shared Memory Driver (SMD)" + depends on MAILBOX + depends on QCOM_SMEM + select RPMSG + help + Say y here to enable support for the Qualcomm Shared Memory Driver + providing communication channels to remote processors in Qualcomm + platforms. + +config RPMSG_VIRTIO + tristate "Virtio RPMSG bus driver" + depends on HAS_DMA + select RPMSG + select RPMSG_NS + select VIRTIO + +endmenu diff --git a/drivers/rpmsg/Makefile b/drivers/rpmsg/Makefile new file mode 100644 index 0000000000..58e3b382e3 --- /dev/null +++ b/drivers/rpmsg/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_RPMSG) += rpmsg_core.o +obj-$(CONFIG_RPMSG_CHAR) += rpmsg_char.o +obj-$(CONFIG_RPMSG_CTRL) += rpmsg_ctrl.o +obj-$(CONFIG_RPMSG_NS) += rpmsg_ns.o +obj-$(CONFIG_RPMSG_MTK_SCP) += mtk_rpmsg.o +qcom_glink-objs := qcom_glink_native.o qcom_glink_ssr.o +obj-$(CONFIG_RPMSG_QCOM_GLINK) += qcom_glink.o +obj-$(CONFIG_RPMSG_QCOM_GLINK_RPM) += qcom_glink_rpm.o +obj-$(CONFIG_RPMSG_QCOM_GLINK_SMEM) += qcom_glink_smem.o +obj-$(CONFIG_RPMSG_QCOM_SMD) += qcom_smd.o +obj-$(CONFIG_RPMSG_VIRTIO) += virtio_rpmsg_bus.o diff --git a/drivers/rpmsg/mtk_rpmsg.c b/drivers/rpmsg/mtk_rpmsg.c new file mode 100644 index 0000000000..d1213c33da --- /dev/null +++ b/drivers/rpmsg/mtk_rpmsg.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright 2019 Google LLC. + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/remoteproc.h> +#include <linux/rpmsg/mtk_rpmsg.h> +#include <linux/slab.h> +#include <linux/workqueue.h> + +#include "rpmsg_internal.h" + +struct mtk_rpmsg_rproc_subdev { + struct platform_device *pdev; + struct mtk_rpmsg_info *info; + struct rpmsg_endpoint *ns_ept; + struct rproc_subdev subdev; + + struct work_struct register_work; + struct list_head channels; + struct mutex channels_lock; +}; + +#define to_mtk_subdev(d) container_of(d, struct mtk_rpmsg_rproc_subdev, subdev) + +struct mtk_rpmsg_channel_info { + struct rpmsg_channel_info info; + bool registered; + struct list_head list; +}; + +/** + * struct rpmsg_ns_msg - dynamic name service announcement message + * @name: name of remote service that is published + * @addr: address of remote service that is published + * + * This message is sent across to publish a new service. When we receive these + * messages, an appropriate rpmsg channel (i.e device) is created. In turn, the + * ->probe() handler of the appropriate rpmsg driver will be invoked + * (if/as-soon-as one is registered). + */ +struct rpmsg_ns_msg { + char name[RPMSG_NAME_SIZE]; + u32 addr; +} __packed; + +struct mtk_rpmsg_device { + struct rpmsg_device rpdev; + struct mtk_rpmsg_rproc_subdev *mtk_subdev; +}; + +struct mtk_rpmsg_endpoint { + struct rpmsg_endpoint ept; + struct mtk_rpmsg_rproc_subdev *mtk_subdev; +}; + +#define to_mtk_rpmsg_device(r) container_of(r, struct mtk_rpmsg_device, rpdev) +#define to_mtk_rpmsg_endpoint(r) container_of(r, struct mtk_rpmsg_endpoint, ept) + +static const struct rpmsg_endpoint_ops mtk_rpmsg_endpoint_ops; + +static void __mtk_ept_release(struct kref *kref) +{ + struct rpmsg_endpoint *ept = container_of(kref, struct rpmsg_endpoint, + refcount); + kfree(to_mtk_rpmsg_endpoint(ept)); +} + +static void mtk_rpmsg_ipi_handler(void *data, unsigned int len, void *priv) +{ + struct mtk_rpmsg_endpoint *mept = priv; + struct rpmsg_endpoint *ept = &mept->ept; + int ret; + + ret = (*ept->cb)(ept->rpdev, data, len, ept->priv, ept->addr); + if (ret) + dev_warn(&ept->rpdev->dev, "rpmsg handler return error = %d", + ret); +} + +static struct rpmsg_endpoint * +__mtk_create_ept(struct mtk_rpmsg_rproc_subdev *mtk_subdev, + struct rpmsg_device *rpdev, rpmsg_rx_cb_t cb, void *priv, + u32 id) +{ + struct mtk_rpmsg_endpoint *mept; + struct rpmsg_endpoint *ept; + struct platform_device *pdev = mtk_subdev->pdev; + int ret; + + mept = kzalloc(sizeof(*mept), GFP_KERNEL); + if (!mept) + return NULL; + mept->mtk_subdev = mtk_subdev; + + ept = &mept->ept; + kref_init(&ept->refcount); + + ept->rpdev = rpdev; + ept->cb = cb; + ept->priv = priv; + ept->ops = &mtk_rpmsg_endpoint_ops; + ept->addr = id; + + ret = mtk_subdev->info->register_ipi(pdev, id, mtk_rpmsg_ipi_handler, + mept); + if (ret) { + dev_err(&pdev->dev, "IPI register failed, id = %d", id); + kref_put(&ept->refcount, __mtk_ept_release); + return NULL; + } + + return ept; +} + +static struct rpmsg_endpoint * +mtk_rpmsg_create_ept(struct rpmsg_device *rpdev, rpmsg_rx_cb_t cb, void *priv, + struct rpmsg_channel_info chinfo) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = + to_mtk_rpmsg_device(rpdev)->mtk_subdev; + + return __mtk_create_ept(mtk_subdev, rpdev, cb, priv, chinfo.src); +} + +static void mtk_rpmsg_destroy_ept(struct rpmsg_endpoint *ept) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = + to_mtk_rpmsg_endpoint(ept)->mtk_subdev; + + mtk_subdev->info->unregister_ipi(mtk_subdev->pdev, ept->addr); + kref_put(&ept->refcount, __mtk_ept_release); +} + +static int mtk_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = + to_mtk_rpmsg_endpoint(ept)->mtk_subdev; + + return mtk_subdev->info->send_ipi(mtk_subdev->pdev, ept->addr, data, + len, 0); +} + +static int mtk_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = + to_mtk_rpmsg_endpoint(ept)->mtk_subdev; + + /* + * TODO: This currently is same as mtk_rpmsg_send, and wait until SCP + * received the last command. + */ + return mtk_subdev->info->send_ipi(mtk_subdev->pdev, ept->addr, data, + len, 0); +} + +static const struct rpmsg_endpoint_ops mtk_rpmsg_endpoint_ops = { + .destroy_ept = mtk_rpmsg_destroy_ept, + .send = mtk_rpmsg_send, + .trysend = mtk_rpmsg_trysend, +}; + +static void mtk_rpmsg_release_device(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct mtk_rpmsg_device *mdev = to_mtk_rpmsg_device(rpdev); + + kfree(mdev); +} + +static const struct rpmsg_device_ops mtk_rpmsg_device_ops = { + .create_ept = mtk_rpmsg_create_ept, +}; + +static struct device_node * +mtk_rpmsg_match_device_subnode(struct device_node *node, const char *channel) +{ + struct device_node *child; + const char *name; + int ret; + + for_each_available_child_of_node(node, child) { + ret = of_property_read_string(child, "mediatek,rpmsg-name", &name); + if (ret) + continue; + + if (strcmp(name, channel) == 0) + return child; + } + + return NULL; +} + +static int mtk_rpmsg_register_device(struct mtk_rpmsg_rproc_subdev *mtk_subdev, + struct rpmsg_channel_info *info) +{ + struct rpmsg_device *rpdev; + struct mtk_rpmsg_device *mdev; + struct platform_device *pdev = mtk_subdev->pdev; + + mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); + if (!mdev) + return -ENOMEM; + + mdev->mtk_subdev = mtk_subdev; + + rpdev = &mdev->rpdev; + rpdev->ops = &mtk_rpmsg_device_ops; + rpdev->src = info->src; + rpdev->dst = info->dst; + strscpy(rpdev->id.name, info->name, RPMSG_NAME_SIZE); + + rpdev->dev.of_node = + mtk_rpmsg_match_device_subnode(pdev->dev.of_node, info->name); + rpdev->dev.parent = &pdev->dev; + rpdev->dev.release = mtk_rpmsg_release_device; + + return rpmsg_register_device(rpdev); +} + +static void mtk_register_device_work_function(struct work_struct *register_work) +{ + struct mtk_rpmsg_rproc_subdev *subdev = container_of( + register_work, struct mtk_rpmsg_rproc_subdev, register_work); + struct platform_device *pdev = subdev->pdev; + struct mtk_rpmsg_channel_info *info; + int ret; + + mutex_lock(&subdev->channels_lock); + list_for_each_entry(info, &subdev->channels, list) { + if (info->registered) + continue; + + mutex_unlock(&subdev->channels_lock); + ret = mtk_rpmsg_register_device(subdev, &info->info); + mutex_lock(&subdev->channels_lock); + if (ret) { + dev_err(&pdev->dev, "Can't create rpmsg_device\n"); + continue; + } + + info->registered = true; + } + mutex_unlock(&subdev->channels_lock); +} + +static int mtk_rpmsg_create_device(struct mtk_rpmsg_rproc_subdev *mtk_subdev, + char *name, u32 addr) +{ + struct mtk_rpmsg_channel_info *info; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + strscpy(info->info.name, name, RPMSG_NAME_SIZE); + info->info.src = addr; + info->info.dst = RPMSG_ADDR_ANY; + mutex_lock(&mtk_subdev->channels_lock); + list_add(&info->list, &mtk_subdev->channels); + mutex_unlock(&mtk_subdev->channels_lock); + + schedule_work(&mtk_subdev->register_work); + return 0; +} + +static int mtk_rpmsg_ns_cb(struct rpmsg_device *rpdev, void *data, int len, + void *priv, u32 src) +{ + struct rpmsg_ns_msg *msg = data; + struct mtk_rpmsg_rproc_subdev *mtk_subdev = priv; + struct device *dev = &mtk_subdev->pdev->dev; + + int ret; + + if (len != sizeof(*msg)) { + dev_err(dev, "malformed ns msg (%d)\n", len); + return -EINVAL; + } + + /* + * the name service ept does _not_ belong to a real rpmsg channel, + * and is handled by the rpmsg bus itself. + * for sanity reasons, make sure a valid rpdev has _not_ sneaked + * in somehow. + */ + if (rpdev) { + dev_err(dev, "anomaly: ns ept has an rpdev handle\n"); + return -EINVAL; + } + + /* don't trust the remote processor for null terminating the name */ + msg->name[RPMSG_NAME_SIZE - 1] = '\0'; + + dev_info(dev, "creating channel %s addr 0x%x\n", msg->name, msg->addr); + + ret = mtk_rpmsg_create_device(mtk_subdev, msg->name, msg->addr); + if (ret) { + dev_err(dev, "create rpmsg device failed\n"); + return ret; + } + + return 0; +} + +static int mtk_rpmsg_prepare(struct rproc_subdev *subdev) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = to_mtk_subdev(subdev); + + /* a dedicated endpoint handles the name service msgs */ + if (mtk_subdev->info->ns_ipi_id >= 0) { + mtk_subdev->ns_ept = + __mtk_create_ept(mtk_subdev, NULL, mtk_rpmsg_ns_cb, + mtk_subdev, + mtk_subdev->info->ns_ipi_id); + if (!mtk_subdev->ns_ept) { + dev_err(&mtk_subdev->pdev->dev, + "failed to create name service endpoint\n"); + return -ENOMEM; + } + } + + return 0; +} + +static void mtk_rpmsg_unprepare(struct rproc_subdev *subdev) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = to_mtk_subdev(subdev); + + if (mtk_subdev->ns_ept) { + mtk_rpmsg_destroy_ept(mtk_subdev->ns_ept); + mtk_subdev->ns_ept = NULL; + } +} + +static void mtk_rpmsg_stop(struct rproc_subdev *subdev, bool crashed) +{ + struct mtk_rpmsg_channel_info *info, *next; + struct mtk_rpmsg_rproc_subdev *mtk_subdev = to_mtk_subdev(subdev); + struct device *dev = &mtk_subdev->pdev->dev; + + /* + * Destroy the name service endpoint here, to avoid new channel being + * created after the rpmsg_unregister_device loop below. + */ + if (mtk_subdev->ns_ept) { + mtk_rpmsg_destroy_ept(mtk_subdev->ns_ept); + mtk_subdev->ns_ept = NULL; + } + + cancel_work_sync(&mtk_subdev->register_work); + + mutex_lock(&mtk_subdev->channels_lock); + list_for_each_entry(info, &mtk_subdev->channels, list) { + if (!info->registered) + continue; + if (rpmsg_unregister_device(dev, &info->info)) { + dev_warn( + dev, + "rpmsg_unregister_device failed for %s.%d.%d\n", + info->info.name, info->info.src, + info->info.dst); + } + } + + list_for_each_entry_safe(info, next, + &mtk_subdev->channels, list) { + list_del(&info->list); + kfree(info); + } + mutex_unlock(&mtk_subdev->channels_lock); +} + +struct rproc_subdev * +mtk_rpmsg_create_rproc_subdev(struct platform_device *pdev, + struct mtk_rpmsg_info *info) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev; + + mtk_subdev = kzalloc(sizeof(*mtk_subdev), GFP_KERNEL); + if (!mtk_subdev) + return NULL; + + mtk_subdev->pdev = pdev; + mtk_subdev->subdev.prepare = mtk_rpmsg_prepare; + mtk_subdev->subdev.stop = mtk_rpmsg_stop; + mtk_subdev->subdev.unprepare = mtk_rpmsg_unprepare; + mtk_subdev->info = info; + INIT_LIST_HEAD(&mtk_subdev->channels); + INIT_WORK(&mtk_subdev->register_work, + mtk_register_device_work_function); + mutex_init(&mtk_subdev->channels_lock); + + return &mtk_subdev->subdev; +} +EXPORT_SYMBOL_GPL(mtk_rpmsg_create_rproc_subdev); + +void mtk_rpmsg_destroy_rproc_subdev(struct rproc_subdev *subdev) +{ + struct mtk_rpmsg_rproc_subdev *mtk_subdev = to_mtk_subdev(subdev); + + kfree(mtk_subdev); +} +EXPORT_SYMBOL_GPL(mtk_rpmsg_destroy_rproc_subdev); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("MediaTek scp rpmsg driver"); diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c new file mode 100644 index 0000000000..82d460ff47 --- /dev/null +++ b/drivers/rpmsg/qcom_glink_native.c @@ -0,0 +1,1882 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016-2017, Linaro Ltd + */ + +#include <linux/idr.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/list.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/rpmsg.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/wait.h> +#include <linux/workqueue.h> +#include <linux/mailbox_client.h> + +#include "rpmsg_internal.h" +#include "qcom_glink_native.h" + +#define GLINK_NAME_SIZE 32 +#define GLINK_VERSION_1 1 + +#define RPM_GLINK_CID_MIN 1 +#define RPM_GLINK_CID_MAX 65536 + +struct glink_msg { + __le16 cmd; + __le16 param1; + __le32 param2; + u8 data[]; +} __packed; + +/** + * struct glink_defer_cmd - deferred incoming control message + * @node: list node + * @msg: message header + * @data: payload of the message + * + * Copy of a received control message, to be added to @rx_queue and processed + * by @rx_work of @qcom_glink. + */ +struct glink_defer_cmd { + struct list_head node; + + struct glink_msg msg; + u8 data[]; +}; + +/** + * struct glink_core_rx_intent - RX intent + * RX intent + * + * @data: pointer to the data (may be NULL for zero-copy) + * @id: remote or local intent ID + * @size: size of the original intent (do not modify) + * @reuse: To mark if the intent can be reused after first use + * @in_use: To mark if intent is already in use for the channel + * @offset: next write offset (initially 0) + * @node: list node + */ +struct glink_core_rx_intent { + void *data; + u32 id; + size_t size; + bool reuse; + bool in_use; + u32 offset; + + struct list_head node; +}; + +/** + * struct qcom_glink - driver context, relates to one remote subsystem + * @dev: reference to the associated struct device + * @rx_pipe: pipe object for receive FIFO + * @tx_pipe: pipe object for transmit FIFO + * @rx_work: worker for handling received control messages + * @rx_lock: protects the @rx_queue + * @rx_queue: queue of received control messages to be processed in @rx_work + * @tx_lock: synchronizes operations on the tx fifo + * @idr_lock: synchronizes @lcids and @rcids modifications + * @lcids: idr of all channels with a known local channel id + * @rcids: idr of all channels with a known remote channel id + * @features: remote features + * @intentless: flag to indicate that there is no intent + * @tx_avail_notify: Waitqueue for pending tx tasks + * @sent_read_notify: flag to check cmd sent or not + * @abort_tx: flag indicating that all tx attempts should fail + */ +struct qcom_glink { + struct device *dev; + + struct qcom_glink_pipe *rx_pipe; + struct qcom_glink_pipe *tx_pipe; + + struct work_struct rx_work; + spinlock_t rx_lock; + struct list_head rx_queue; + + spinlock_t tx_lock; + + spinlock_t idr_lock; + struct idr lcids; + struct idr rcids; + unsigned long features; + + bool intentless; + wait_queue_head_t tx_avail_notify; + bool sent_read_notify; + + bool abort_tx; +}; + +enum { + GLINK_STATE_CLOSED, + GLINK_STATE_OPENING, + GLINK_STATE_OPEN, + GLINK_STATE_CLOSING, +}; + +/** + * struct glink_channel - internal representation of a channel + * @rpdev: rpdev reference, only used for primary endpoints + * @ept: rpmsg endpoint this channel is associated with + * @glink: qcom_glink context handle + * @refcount: refcount for the channel object + * @recv_lock: guard for @ept.cb + * @name: unique channel name/identifier + * @lcid: channel id, in local space + * @rcid: channel id, in remote space + * @intent_lock: lock for protection of @liids, @riids + * @liids: idr of all local intents + * @riids: idr of all remote intents + * @intent_work: worker responsible for transmitting rx_done packets + * @done_intents: list of intents that needs to be announced rx_done + * @buf: receive buffer, for gathering fragments + * @buf_offset: write offset in @buf + * @buf_size: size of current @buf + * @open_ack: completed once remote has acked the open-request + * @open_req: completed once open-request has been received + * @intent_req_lock: Synchronises multiple intent requests + * @intent_req_result: Result of intent request + * @intent_received: flag indicating that an intent has been received + * @intent_req_wq: wait queue for intent_req signalling + */ +struct glink_channel { + struct rpmsg_endpoint ept; + + struct rpmsg_device *rpdev; + struct qcom_glink *glink; + + struct kref refcount; + + spinlock_t recv_lock; + + char *name; + unsigned int lcid; + unsigned int rcid; + + spinlock_t intent_lock; + struct idr liids; + struct idr riids; + struct work_struct intent_work; + struct list_head done_intents; + + struct glink_core_rx_intent *buf; + int buf_offset; + int buf_size; + + struct completion open_ack; + struct completion open_req; + + struct mutex intent_req_lock; + int intent_req_result; + bool intent_received; + wait_queue_head_t intent_req_wq; +}; + +#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) + +static const struct rpmsg_endpoint_ops glink_endpoint_ops; + +#define GLINK_CMD_VERSION 0 +#define GLINK_CMD_VERSION_ACK 1 +#define GLINK_CMD_OPEN 2 +#define GLINK_CMD_CLOSE 3 +#define GLINK_CMD_OPEN_ACK 4 +#define GLINK_CMD_INTENT 5 +#define GLINK_CMD_RX_DONE 6 +#define GLINK_CMD_RX_INTENT_REQ 7 +#define GLINK_CMD_RX_INTENT_REQ_ACK 8 +#define GLINK_CMD_TX_DATA 9 +#define GLINK_CMD_CLOSE_ACK 11 +#define GLINK_CMD_TX_DATA_CONT 12 +#define GLINK_CMD_READ_NOTIF 13 +#define GLINK_CMD_RX_DONE_W_REUSE 14 +#define GLINK_CMD_SIGNALS 15 + +#define GLINK_FEATURE_INTENTLESS BIT(1) + +#define NATIVE_DTR_SIG NATIVE_DSR_SIG +#define NATIVE_DSR_SIG BIT(31) +#define NATIVE_RTS_SIG NATIVE_CTS_SIG +#define NATIVE_CTS_SIG BIT(30) + +static void qcom_glink_rx_done_work(struct work_struct *work); + +static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink, + const char *name) +{ + struct glink_channel *channel; + + channel = kzalloc(sizeof(*channel), GFP_KERNEL); + if (!channel) + return ERR_PTR(-ENOMEM); + + /* Setup glink internal glink_channel data */ + spin_lock_init(&channel->recv_lock); + spin_lock_init(&channel->intent_lock); + mutex_init(&channel->intent_req_lock); + + channel->glink = glink; + channel->name = kstrdup(name, GFP_KERNEL); + if (!channel->name) { + kfree(channel); + return ERR_PTR(-ENOMEM); + } + + init_completion(&channel->open_req); + init_completion(&channel->open_ack); + init_waitqueue_head(&channel->intent_req_wq); + + INIT_LIST_HEAD(&channel->done_intents); + INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work); + + idr_init(&channel->liids); + idr_init(&channel->riids); + kref_init(&channel->refcount); + + return channel; +} + +static void qcom_glink_channel_release(struct kref *ref) +{ + struct glink_channel *channel = container_of(ref, struct glink_channel, + refcount); + struct glink_core_rx_intent *intent; + struct glink_core_rx_intent *tmp; + unsigned long flags; + int iid; + + /* cancel pending rx_done work */ + cancel_work_sync(&channel->intent_work); + + spin_lock_irqsave(&channel->intent_lock, flags); + /* Free all non-reuse intents pending rx_done work */ + list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { + if (!intent->reuse) { + kfree(intent->data); + kfree(intent); + } + } + + idr_for_each_entry(&channel->liids, tmp, iid) { + kfree(tmp->data); + kfree(tmp); + } + idr_destroy(&channel->liids); + + idr_for_each_entry(&channel->riids, tmp, iid) + kfree(tmp); + idr_destroy(&channel->riids); + spin_unlock_irqrestore(&channel->intent_lock, flags); + + kfree(channel->name); + kfree(channel); +} + +static size_t qcom_glink_rx_avail(struct qcom_glink *glink) +{ + return glink->rx_pipe->avail(glink->rx_pipe); +} + +static void qcom_glink_rx_peek(struct qcom_glink *glink, + void *data, unsigned int offset, size_t count) +{ + glink->rx_pipe->peek(glink->rx_pipe, data, offset, count); +} + +static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count) +{ + glink->rx_pipe->advance(glink->rx_pipe, count); +} + +static size_t qcom_glink_tx_avail(struct qcom_glink *glink) +{ + return glink->tx_pipe->avail(glink->tx_pipe); +} + +static void qcom_glink_tx_write(struct qcom_glink *glink, + const void *hdr, size_t hlen, + const void *data, size_t dlen) +{ + glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen); +} + +static void qcom_glink_tx_kick(struct qcom_glink *glink) +{ + glink->tx_pipe->kick(glink->tx_pipe); +} + +static void qcom_glink_send_read_notify(struct qcom_glink *glink) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF); + msg.param1 = 0; + msg.param2 = 0; + + qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0); + + qcom_glink_tx_kick(glink); +} + +static int qcom_glink_tx(struct qcom_glink *glink, + const void *hdr, size_t hlen, + const void *data, size_t dlen, bool wait) +{ + unsigned int tlen = hlen + dlen; + unsigned long flags; + int ret = 0; + + /* Reject packets that are too big */ + if (tlen >= glink->tx_pipe->length) + return -EINVAL; + + spin_lock_irqsave(&glink->tx_lock, flags); + + if (glink->abort_tx) { + ret = -EIO; + goto out; + } + + while (qcom_glink_tx_avail(glink) < tlen) { + if (!wait) { + ret = -EAGAIN; + goto out; + } + + if (glink->abort_tx) { + ret = -EIO; + goto out; + } + + if (!glink->sent_read_notify) { + glink->sent_read_notify = true; + qcom_glink_send_read_notify(glink); + } + + /* Wait without holding the tx_lock */ + spin_unlock_irqrestore(&glink->tx_lock, flags); + + wait_event_timeout(glink->tx_avail_notify, + qcom_glink_tx_avail(glink) >= tlen, 10 * HZ); + + spin_lock_irqsave(&glink->tx_lock, flags); + + if (qcom_glink_tx_avail(glink) >= tlen) + glink->sent_read_notify = false; + } + + qcom_glink_tx_write(glink, hdr, hlen, data, dlen); + qcom_glink_tx_kick(glink); + +out: + spin_unlock_irqrestore(&glink->tx_lock, flags); + + return ret; +} + +static int qcom_glink_send_version(struct qcom_glink *glink) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(GLINK_CMD_VERSION); + msg.param1 = cpu_to_le16(GLINK_VERSION_1); + msg.param2 = cpu_to_le32(glink->features); + + return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); +} + +static void qcom_glink_send_version_ack(struct qcom_glink *glink) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK); + msg.param1 = cpu_to_le16(GLINK_VERSION_1); + msg.param2 = cpu_to_le32(glink->features); + + qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); +} + +static void qcom_glink_send_open_ack(struct qcom_glink *glink, + struct glink_channel *channel) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK); + msg.param1 = cpu_to_le16(channel->rcid); + msg.param2 = cpu_to_le32(0); + + qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); +} + +static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink, + unsigned int cid, bool granted) +{ + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, cid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_err(glink->dev, "unable to find channel\n"); + return; + } + + WRITE_ONCE(channel->intent_req_result, granted); + wake_up_all(&channel->intent_req_wq); +} + +static void qcom_glink_intent_req_abort(struct glink_channel *channel) +{ + WRITE_ONCE(channel->intent_req_result, 0); + wake_up_all(&channel->intent_req_wq); +} + +/** + * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote + * @glink: Ptr to the glink edge + * @channel: Ptr to the channel that the open req is sent + * + * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote. + * Will return with refcount held, regardless of outcome. + * + * Return: 0 on success, negative errno otherwise. + */ +static int qcom_glink_send_open_req(struct qcom_glink *glink, + struct glink_channel *channel) +{ + struct { + struct glink_msg msg; + u8 name[GLINK_NAME_SIZE]; + } __packed req; + int name_len = strlen(channel->name) + 1; + int req_len = ALIGN(sizeof(req.msg) + name_len, 8); + int ret; + unsigned long flags; + + kref_get(&channel->refcount); + + spin_lock_irqsave(&glink->idr_lock, flags); + ret = idr_alloc_cyclic(&glink->lcids, channel, + RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, + GFP_ATOMIC); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (ret < 0) + return ret; + + channel->lcid = ret; + + req.msg.cmd = cpu_to_le16(GLINK_CMD_OPEN); + req.msg.param1 = cpu_to_le16(channel->lcid); + req.msg.param2 = cpu_to_le32(name_len); + strcpy(req.name, channel->name); + + ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true); + if (ret) + goto remove_idr; + + return 0; + +remove_idr: + spin_lock_irqsave(&glink->idr_lock, flags); + idr_remove(&glink->lcids, channel->lcid); + channel->lcid = 0; + spin_unlock_irqrestore(&glink->idr_lock, flags); + + return ret; +} + +static void qcom_glink_send_close_req(struct qcom_glink *glink, + struct glink_channel *channel) +{ + struct glink_msg req; + + req.cmd = cpu_to_le16(GLINK_CMD_CLOSE); + req.param1 = cpu_to_le16(channel->lcid); + req.param2 = 0; + + qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); +} + +static void qcom_glink_send_close_ack(struct qcom_glink *glink, + unsigned int rcid) +{ + struct glink_msg req; + + req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK); + req.param1 = cpu_to_le16(rcid); + req.param2 = 0; + + qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); +} + +static void qcom_glink_rx_done_work(struct work_struct *work) +{ + struct glink_channel *channel = container_of(work, struct glink_channel, + intent_work); + struct qcom_glink *glink = channel->glink; + struct glink_core_rx_intent *intent, *tmp; + struct { + u16 id; + u16 lcid; + u32 liid; + } __packed cmd; + + unsigned int cid = channel->lcid; + unsigned int iid; + bool reuse; + unsigned long flags; + + spin_lock_irqsave(&channel->intent_lock, flags); + list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { + list_del(&intent->node); + spin_unlock_irqrestore(&channel->intent_lock, flags); + iid = intent->id; + reuse = intent->reuse; + + cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE; + cmd.lcid = cid; + cmd.liid = iid; + + qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); + if (!reuse) { + kfree(intent->data); + kfree(intent); + } + spin_lock_irqsave(&channel->intent_lock, flags); + } + spin_unlock_irqrestore(&channel->intent_lock, flags); +} + +static void qcom_glink_rx_done(struct qcom_glink *glink, + struct glink_channel *channel, + struct glink_core_rx_intent *intent) +{ + /* We don't send RX_DONE to intentless systems */ + if (glink->intentless) { + kfree(intent->data); + kfree(intent); + return; + } + + /* Take it off the tree of receive intents */ + if (!intent->reuse) { + spin_lock(&channel->intent_lock); + idr_remove(&channel->liids, intent->id); + spin_unlock(&channel->intent_lock); + } + + /* Schedule the sending of a rx_done indication */ + spin_lock(&channel->intent_lock); + list_add_tail(&intent->node, &channel->done_intents); + spin_unlock(&channel->intent_lock); + + schedule_work(&channel->intent_work); +} + +/** + * qcom_glink_receive_version() - receive version/features from remote system + * + * @glink: pointer to transport interface + * @version: remote version + * @features: remote features + * + * This function is called in response to a remote-initiated version/feature + * negotiation sequence. + */ +static void qcom_glink_receive_version(struct qcom_glink *glink, + u32 version, + u32 features) +{ + switch (version) { + case 0: + break; + case GLINK_VERSION_1: + glink->features &= features; + fallthrough; + default: + qcom_glink_send_version_ack(glink); + break; + } +} + +/** + * qcom_glink_receive_version_ack() - receive negotiation ack from remote system + * + * @glink: pointer to transport interface + * @version: remote version response + * @features: remote features response + * + * This function is called in response to a local-initiated version/feature + * negotiation sequence and is the counter-offer from the remote side based + * upon the initial version and feature set requested. + */ +static void qcom_glink_receive_version_ack(struct qcom_glink *glink, + u32 version, + u32 features) +{ + switch (version) { + case 0: + /* Version negotiation failed */ + break; + case GLINK_VERSION_1: + if (features == glink->features) + break; + + glink->features &= features; + fallthrough; + default: + qcom_glink_send_version(glink); + break; + } +} + +/** + * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to + * wire format and transmit + * @glink: The transport to transmit on. + * @channel: The glink channel + * @granted: The request response to encode. + * + * Return: 0 on success or standard Linux error code. + */ +static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink, + struct glink_channel *channel, + bool granted) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK); + msg.param1 = cpu_to_le16(channel->lcid); + msg.param2 = cpu_to_le32(granted); + + qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); + + return 0; +} + +/** + * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and + * transmit + * @glink: The transport to transmit on. + * @channel: The local channel + * @intent: The intent to pass on to remote. + * + * Return: 0 on success or standard Linux error code. + */ +static int qcom_glink_advertise_intent(struct qcom_glink *glink, + struct glink_channel *channel, + struct glink_core_rx_intent *intent) +{ + struct command { + __le16 id; + __le16 lcid; + __le32 count; + __le32 size; + __le32 liid; + } __packed; + struct command cmd; + + cmd.id = cpu_to_le16(GLINK_CMD_INTENT); + cmd.lcid = cpu_to_le16(channel->lcid); + cmd.count = cpu_to_le32(1); + cmd.size = cpu_to_le32(intent->size); + cmd.liid = cpu_to_le32(intent->id); + + qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); + + return 0; +} + +static struct glink_core_rx_intent * +qcom_glink_alloc_intent(struct qcom_glink *glink, + struct glink_channel *channel, + size_t size, + bool reuseable) +{ + struct glink_core_rx_intent *intent; + int ret; + unsigned long flags; + + intent = kzalloc(sizeof(*intent), GFP_KERNEL); + if (!intent) + return NULL; + + intent->data = kzalloc(size, GFP_KERNEL); + if (!intent->data) + goto free_intent; + + spin_lock_irqsave(&channel->intent_lock, flags); + ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC); + if (ret < 0) { + spin_unlock_irqrestore(&channel->intent_lock, flags); + goto free_data; + } + spin_unlock_irqrestore(&channel->intent_lock, flags); + + intent->id = ret; + intent->size = size; + intent->reuse = reuseable; + + return intent; + +free_data: + kfree(intent->data); +free_intent: + kfree(intent); + return NULL; +} + +static void qcom_glink_handle_rx_done(struct qcom_glink *glink, + u32 cid, uint32_t iid, + bool reuse) +{ + struct glink_core_rx_intent *intent; + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, cid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_err(glink->dev, "invalid channel id received\n"); + return; + } + + spin_lock_irqsave(&channel->intent_lock, flags); + intent = idr_find(&channel->riids, iid); + + if (!intent) { + spin_unlock_irqrestore(&channel->intent_lock, flags); + dev_err(glink->dev, "invalid intent id received\n"); + return; + } + + intent->in_use = false; + + if (!reuse) { + idr_remove(&channel->riids, intent->id); + kfree(intent); + } + spin_unlock_irqrestore(&channel->intent_lock, flags); + + if (reuse) { + WRITE_ONCE(channel->intent_received, true); + wake_up_all(&channel->intent_req_wq); + } +} + +/** + * qcom_glink_handle_intent_req() - Receive a request for rx_intent + * from remote side + * @glink: Pointer to the transport interface + * @cid: Remote channel ID + * @size: size of the intent + * + * The function searches for the local channel to which the request for + * rx_intent has arrived and allocates and notifies the remote back + */ +static void qcom_glink_handle_intent_req(struct qcom_glink *glink, + u32 cid, size_t size) +{ + struct glink_core_rx_intent *intent; + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, cid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + + if (!channel) { + pr_err("%s channel not found for cid %d\n", __func__, cid); + return; + } + + intent = qcom_glink_alloc_intent(glink, channel, size, false); + if (intent) + qcom_glink_advertise_intent(glink, channel, intent); + + qcom_glink_send_intent_req_ack(glink, channel, !!intent); +} + +static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra) +{ + struct glink_defer_cmd *dcmd; + + extra = ALIGN(extra, 8); + + if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) { + dev_dbg(glink->dev, "Insufficient data in rx fifo"); + return -ENXIO; + } + + dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC); + if (!dcmd) + return -ENOMEM; + + INIT_LIST_HEAD(&dcmd->node); + + qcom_glink_rx_peek(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra); + + spin_lock(&glink->rx_lock); + list_add_tail(&dcmd->node, &glink->rx_queue); + spin_unlock(&glink->rx_lock); + + schedule_work(&glink->rx_work); + qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra); + + return 0; +} + +static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) +{ + struct glink_core_rx_intent *intent; + struct glink_channel *channel; + struct { + struct glink_msg msg; + __le32 chunk_size; + __le32 left_size; + } __packed hdr; + unsigned int chunk_size; + unsigned int left_size; + unsigned int rcid; + unsigned int liid; + int ret = 0; + unsigned long flags; + + if (avail < sizeof(hdr)) { + dev_dbg(glink->dev, "Not enough data in fifo\n"); + return -EAGAIN; + } + + qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr)); + chunk_size = le32_to_cpu(hdr.chunk_size); + left_size = le32_to_cpu(hdr.left_size); + + if (avail < sizeof(hdr) + chunk_size) { + dev_dbg(glink->dev, "Payload not yet in fifo\n"); + return -EAGAIN; + } + + rcid = le16_to_cpu(hdr.msg.param1); + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, rcid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_dbg(glink->dev, "Data on non-existing channel\n"); + + /* Drop the message */ + goto advance_rx; + } + + if (glink->intentless) { + /* Might have an ongoing, fragmented, message to append */ + if (!channel->buf) { + intent = kzalloc(sizeof(*intent), GFP_ATOMIC); + if (!intent) + return -ENOMEM; + + intent->data = kmalloc(chunk_size + left_size, + GFP_ATOMIC); + if (!intent->data) { + kfree(intent); + return -ENOMEM; + } + + intent->id = 0xbabababa; + intent->size = chunk_size + left_size; + intent->offset = 0; + + channel->buf = intent; + } else { + intent = channel->buf; + } + } else { + liid = le32_to_cpu(hdr.msg.param2); + + spin_lock_irqsave(&channel->intent_lock, flags); + intent = idr_find(&channel->liids, liid); + spin_unlock_irqrestore(&channel->intent_lock, flags); + + if (!intent) { + dev_err(glink->dev, + "no intent found for channel %s intent %d", + channel->name, liid); + ret = -ENOENT; + goto advance_rx; + } + } + + if (intent->size - intent->offset < chunk_size) { + dev_err(glink->dev, "Insufficient space in intent\n"); + + /* The packet header lied, drop payload */ + goto advance_rx; + } + + qcom_glink_rx_peek(glink, intent->data + intent->offset, + sizeof(hdr), chunk_size); + intent->offset += chunk_size; + + /* Handle message when no fragments remain to be received */ + if (!left_size) { + spin_lock(&channel->recv_lock); + if (channel->ept.cb) { + channel->ept.cb(channel->ept.rpdev, + intent->data, + intent->offset, + channel->ept.priv, + RPMSG_ADDR_ANY); + } + spin_unlock(&channel->recv_lock); + + intent->offset = 0; + channel->buf = NULL; + + qcom_glink_rx_done(glink, channel, intent); + } + +advance_rx: + qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); + + return ret; +} + +static void qcom_glink_handle_intent(struct qcom_glink *glink, + unsigned int cid, + unsigned int count, + size_t avail) +{ + struct glink_core_rx_intent *intent; + struct glink_channel *channel; + struct intent_pair { + __le32 size; + __le32 iid; + }; + + struct { + struct glink_msg msg; + struct intent_pair intents[]; + } __packed * msg; + + const size_t msglen = struct_size(msg, intents, count); + int ret; + int i; + unsigned long flags; + + if (avail < msglen) { + dev_dbg(glink->dev, "Not enough data in fifo\n"); + return; + } + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, cid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_err(glink->dev, "intents for non-existing channel\n"); + qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); + return; + } + + msg = kmalloc(msglen, GFP_ATOMIC); + if (!msg) + return; + + qcom_glink_rx_peek(glink, msg, 0, msglen); + + for (i = 0; i < count; ++i) { + intent = kzalloc(sizeof(*intent), GFP_ATOMIC); + if (!intent) + break; + + intent->id = le32_to_cpu(msg->intents[i].iid); + intent->size = le32_to_cpu(msg->intents[i].size); + + spin_lock_irqsave(&channel->intent_lock, flags); + ret = idr_alloc(&channel->riids, intent, + intent->id, intent->id + 1, GFP_ATOMIC); + spin_unlock_irqrestore(&channel->intent_lock, flags); + + if (ret < 0) + dev_err(glink->dev, "failed to store remote intent\n"); + } + + WRITE_ONCE(channel->intent_received, true); + wake_up_all(&channel->intent_req_wq); + + kfree(msg); + qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); +} + +static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) +{ + struct glink_channel *channel; + + spin_lock(&glink->idr_lock); + channel = idr_find(&glink->lcids, lcid); + spin_unlock(&glink->idr_lock); + if (!channel) { + dev_err(glink->dev, "Invalid open ack packet\n"); + return -EINVAL; + } + + complete_all(&channel->open_ack); + + return 0; +} + +/** + * qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit + * @ept: Rpmsg endpoint for channel. + * @pause: Pause transmission + * @dst: destination address of the endpoint + * + * Return: 0 on success or standard Linux error code. + */ +static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst) +{ + struct glink_channel *channel = to_glink_channel(ept); + struct qcom_glink *glink = channel->glink; + struct glink_msg msg; + u32 sigs = 0; + + if (pause) + sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG; + + msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS); + msg.param1 = cpu_to_le16(channel->lcid); + msg.param2 = cpu_to_le32(sigs); + + return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); +} + +static void qcom_glink_handle_signals(struct qcom_glink *glink, + unsigned int rcid, unsigned int sigs) +{ + struct glink_channel *channel; + unsigned long flags; + bool enable; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, rcid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_err(glink->dev, "signal for non-existing channel\n"); + return; + } + + enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG; + + if (channel->ept.flow_cb) + channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable); +} + +void qcom_glink_native_rx(struct qcom_glink *glink) +{ + struct glink_msg msg; + unsigned int param1; + unsigned int param2; + unsigned int avail; + unsigned int cmd; + int ret = 0; + + /* To wakeup any blocking writers */ + wake_up_all(&glink->tx_avail_notify); + + for (;;) { + avail = qcom_glink_rx_avail(glink); + if (avail < sizeof(msg)) + break; + + qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg)); + + cmd = le16_to_cpu(msg.cmd); + param1 = le16_to_cpu(msg.param1); + param2 = le32_to_cpu(msg.param2); + + switch (cmd) { + case GLINK_CMD_VERSION: + case GLINK_CMD_VERSION_ACK: + case GLINK_CMD_CLOSE: + case GLINK_CMD_CLOSE_ACK: + case GLINK_CMD_RX_INTENT_REQ: + ret = qcom_glink_rx_defer(glink, 0); + break; + case GLINK_CMD_OPEN_ACK: + ret = qcom_glink_rx_open_ack(glink, param1); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + case GLINK_CMD_OPEN: + ret = qcom_glink_rx_defer(glink, param2); + break; + case GLINK_CMD_TX_DATA: + case GLINK_CMD_TX_DATA_CONT: + ret = qcom_glink_rx_data(glink, avail); + break; + case GLINK_CMD_READ_NOTIF: + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + qcom_glink_tx_kick(glink); + break; + case GLINK_CMD_INTENT: + qcom_glink_handle_intent(glink, param1, param2, avail); + break; + case GLINK_CMD_RX_DONE: + qcom_glink_handle_rx_done(glink, param1, param2, false); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + case GLINK_CMD_RX_DONE_W_REUSE: + qcom_glink_handle_rx_done(glink, param1, param2, true); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + case GLINK_CMD_RX_INTENT_REQ_ACK: + qcom_glink_handle_intent_req_ack(glink, param1, param2); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + case GLINK_CMD_SIGNALS: + qcom_glink_handle_signals(glink, param1, param2); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + default: + dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); + ret = -EINVAL; + break; + } + + if (ret) + break; + } +} +EXPORT_SYMBOL(qcom_glink_native_rx); + +/* Locally initiated rpmsg_create_ept */ +static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink, + const char *name) +{ + struct glink_channel *channel; + int ret; + unsigned long flags; + + channel = qcom_glink_alloc_channel(glink, name); + if (IS_ERR(channel)) + return ERR_CAST(channel); + + ret = qcom_glink_send_open_req(glink, channel); + if (ret) + goto release_channel; + + ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); + if (!ret) + goto err_timeout; + + ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); + if (!ret) + goto err_timeout; + + qcom_glink_send_open_ack(glink, channel); + + return channel; + +err_timeout: + /* qcom_glink_send_open_req() did register the channel in lcids*/ + spin_lock_irqsave(&glink->idr_lock, flags); + idr_remove(&glink->lcids, channel->lcid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + +release_channel: + /* Release qcom_glink_send_open_req() reference */ + kref_put(&channel->refcount, qcom_glink_channel_release); + /* Release qcom_glink_alloc_channel() reference */ + kref_put(&channel->refcount, qcom_glink_channel_release); + + return ERR_PTR(-ETIMEDOUT); +} + +/* Remote initiated rpmsg_create_ept */ +static int qcom_glink_create_remote(struct qcom_glink *glink, + struct glink_channel *channel) +{ + int ret; + + qcom_glink_send_open_ack(glink, channel); + + ret = qcom_glink_send_open_req(glink, channel); + if (ret) + goto close_link; + + ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); + if (!ret) { + ret = -ETIMEDOUT; + goto close_link; + } + + return 0; + +close_link: + /* + * Send a close request to "undo" our open-ack. The close-ack will + * release qcom_glink_send_open_req() reference and the last reference + * will be relesed after receiving remote_close or transport unregister + * by calling qcom_glink_native_remove(). + */ + qcom_glink_send_close_req(glink, channel); + + return ret; +} + +static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev, + rpmsg_rx_cb_t cb, + void *priv, + struct rpmsg_channel_info + chinfo) +{ + struct glink_channel *parent = to_glink_channel(rpdev->ept); + struct glink_channel *channel; + struct qcom_glink *glink = parent->glink; + struct rpmsg_endpoint *ept; + const char *name = chinfo.name; + int cid; + int ret; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + idr_for_each_entry(&glink->rcids, channel, cid) { + if (!strcmp(channel->name, name)) + break; + } + spin_unlock_irqrestore(&glink->idr_lock, flags); + + if (!channel) { + channel = qcom_glink_create_local(glink, name); + if (IS_ERR(channel)) + return NULL; + } else { + ret = qcom_glink_create_remote(glink, channel); + if (ret) + return NULL; + } + + ept = &channel->ept; + ept->rpdev = rpdev; + ept->cb = cb; + ept->priv = priv; + ept->ops = &glink_endpoint_ops; + + return ept; +} + +static int qcom_glink_announce_create(struct rpmsg_device *rpdev) +{ + struct glink_channel *channel = to_glink_channel(rpdev->ept); + struct device_node *np = rpdev->dev.of_node; + struct qcom_glink *glink = channel->glink; + struct glink_core_rx_intent *intent; + const struct property *prop = NULL; + __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) }; + int num_intents; + int num_groups = 1; + __be32 *val = defaults; + int size; + + if (glink->intentless || !completion_done(&channel->open_ack)) + return 0; + + prop = of_find_property(np, "qcom,intents", NULL); + if (prop) { + val = prop->value; + num_groups = prop->length / sizeof(u32) / 2; + } + + /* Channel is now open, advertise base set of intents */ + while (num_groups--) { + size = be32_to_cpup(val++); + num_intents = be32_to_cpup(val++); + while (num_intents--) { + intent = qcom_glink_alloc_intent(glink, channel, size, + true); + if (!intent) + break; + + qcom_glink_advertise_intent(glink, channel, intent); + } + } + return 0; +} + +static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) +{ + struct glink_channel *channel = to_glink_channel(ept); + struct qcom_glink *glink = channel->glink; + unsigned long flags; + + spin_lock_irqsave(&channel->recv_lock, flags); + channel->ept.cb = NULL; + spin_unlock_irqrestore(&channel->recv_lock, flags); + + /* Decouple the potential rpdev from the channel */ + channel->rpdev = NULL; + + qcom_glink_send_close_req(glink, channel); +} + +static int qcom_glink_request_intent(struct qcom_glink *glink, + struct glink_channel *channel, + size_t size) +{ + struct { + u16 id; + u16 cid; + u32 size; + } __packed cmd; + + int ret; + + mutex_lock(&channel->intent_req_lock); + + WRITE_ONCE(channel->intent_req_result, -1); + WRITE_ONCE(channel->intent_received, false); + + cmd.id = GLINK_CMD_RX_INTENT_REQ; + cmd.cid = channel->lcid; + cmd.size = size; + + ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); + if (ret) + goto unlock; + + ret = wait_event_timeout(channel->intent_req_wq, + READ_ONCE(channel->intent_req_result) >= 0 && + READ_ONCE(channel->intent_received), + 10 * HZ); + if (!ret) { + dev_err(glink->dev, "intent request timed out\n"); + ret = -ETIMEDOUT; + } else { + ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED; + } + +unlock: + mutex_unlock(&channel->intent_req_lock); + return ret; +} + +static int __qcom_glink_send(struct glink_channel *channel, + void *data, int len, bool wait) +{ + struct qcom_glink *glink = channel->glink; + struct glink_core_rx_intent *intent = NULL; + struct glink_core_rx_intent *tmp; + int iid = 0; + struct { + struct glink_msg msg; + __le32 chunk_size; + __le32 left_size; + } __packed req; + int ret; + unsigned long flags; + int chunk_size = len; + size_t offset = 0; + + if (!glink->intentless) { + while (!intent) { + spin_lock_irqsave(&channel->intent_lock, flags); + idr_for_each_entry(&channel->riids, tmp, iid) { + if (tmp->size >= len && !tmp->in_use) { + if (!intent) + intent = tmp; + else if (intent->size > tmp->size) + intent = tmp; + if (intent->size == len) + break; + } + } + if (intent) + intent->in_use = true; + spin_unlock_irqrestore(&channel->intent_lock, flags); + + /* We found an available intent */ + if (intent) + break; + + if (!wait) + return -EBUSY; + + ret = qcom_glink_request_intent(glink, channel, len); + if (ret < 0) + return ret; + } + + iid = intent->id; + } + + while (offset < len) { + chunk_size = len - offset; + if (chunk_size > SZ_8K && wait) + chunk_size = SZ_8K; + + req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT); + req.msg.param1 = cpu_to_le16(channel->lcid); + req.msg.param2 = cpu_to_le32(iid); + req.chunk_size = cpu_to_le32(chunk_size); + req.left_size = cpu_to_le32(len - offset - chunk_size); + + ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait); + if (ret) { + /* Mark intent available if we failed */ + if (intent) + intent->in_use = false; + return ret; + } + + offset += chunk_size; + } + + return 0; +} + +static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct glink_channel *channel = to_glink_channel(ept); + + return __qcom_glink_send(channel, data, len, true); +} + +static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct glink_channel *channel = to_glink_channel(ept); + + return __qcom_glink_send(channel, data, len, false); +} + +static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) +{ + struct glink_channel *channel = to_glink_channel(ept); + + return __qcom_glink_send(channel, data, len, true); +} + +static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) +{ + struct glink_channel *channel = to_glink_channel(ept); + + return __qcom_glink_send(channel, data, len, false); +} + +/* + * Finds the device_node for the glink child interested in this channel. + */ +static struct device_node *qcom_glink_match_channel(struct device_node *node, + const char *channel) +{ + struct device_node *child; + const char *name; + const char *key; + int ret; + + for_each_available_child_of_node(node, child) { + key = "qcom,glink-channels"; + ret = of_property_read_string(child, key, &name); + if (ret) + continue; + + if (strcmp(name, channel) == 0) + return child; + } + + return NULL; +} + +static const struct rpmsg_device_ops glink_device_ops = { + .create_ept = qcom_glink_create_ept, + .announce_create = qcom_glink_announce_create, +}; + +static const struct rpmsg_endpoint_ops glink_endpoint_ops = { + .destroy_ept = qcom_glink_destroy_ept, + .send = qcom_glink_send, + .sendto = qcom_glink_sendto, + .trysend = qcom_glink_trysend, + .trysendto = qcom_glink_trysendto, + .set_flow_control = qcom_glink_set_flow_control, +}; + +static void qcom_glink_rpdev_release(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + + kfree(rpdev->driver_override); + kfree(rpdev); +} + +static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, + char *name) +{ + struct glink_channel *channel; + struct rpmsg_device *rpdev; + bool create_device = false; + struct device_node *node; + int lcid; + int ret; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + idr_for_each_entry(&glink->lcids, channel, lcid) { + if (!strcmp(channel->name, name)) + break; + } + spin_unlock_irqrestore(&glink->idr_lock, flags); + + if (!channel) { + channel = qcom_glink_alloc_channel(glink, name); + if (IS_ERR(channel)) + return PTR_ERR(channel); + + /* The opening dance was initiated by the remote */ + create_device = true; + } + + spin_lock_irqsave(&glink->idr_lock, flags); + ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC); + if (ret < 0) { + dev_err(glink->dev, "Unable to insert channel into rcid list\n"); + spin_unlock_irqrestore(&glink->idr_lock, flags); + goto free_channel; + } + channel->rcid = ret; + spin_unlock_irqrestore(&glink->idr_lock, flags); + + complete_all(&channel->open_req); + + if (create_device) { + rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); + if (!rpdev) { + ret = -ENOMEM; + goto rcid_remove; + } + + rpdev->ept = &channel->ept; + strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE); + rpdev->src = RPMSG_ADDR_ANY; + rpdev->dst = RPMSG_ADDR_ANY; + rpdev->ops = &glink_device_ops; + + node = qcom_glink_match_channel(glink->dev->of_node, name); + rpdev->dev.of_node = node; + rpdev->dev.parent = glink->dev; + rpdev->dev.release = qcom_glink_rpdev_release; + + ret = rpmsg_register_device(rpdev); + if (ret) + goto rcid_remove; + + channel->rpdev = rpdev; + } + + return 0; + +rcid_remove: + spin_lock_irqsave(&glink->idr_lock, flags); + idr_remove(&glink->rcids, channel->rcid); + channel->rcid = 0; + spin_unlock_irqrestore(&glink->idr_lock, flags); +free_channel: + /* Release the reference, iff we took it */ + if (create_device) + kref_put(&channel->refcount, qcom_glink_channel_release); + + return ret; +} + +static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) +{ + struct rpmsg_channel_info chinfo; + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, rcid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (WARN(!channel, "close request on unknown channel\n")) + return; + + /* cancel pending rx_done work */ + cancel_work_sync(&channel->intent_work); + + if (channel->rpdev) { + strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); + chinfo.src = RPMSG_ADDR_ANY; + chinfo.dst = RPMSG_ADDR_ANY; + + rpmsg_unregister_device(glink->dev, &chinfo); + } + channel->rpdev = NULL; + + qcom_glink_send_close_ack(glink, channel->rcid); + + spin_lock_irqsave(&glink->idr_lock, flags); + idr_remove(&glink->rcids, channel->rcid); + channel->rcid = 0; + spin_unlock_irqrestore(&glink->idr_lock, flags); + + kref_put(&channel->refcount, qcom_glink_channel_release); +} + +static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) +{ + struct rpmsg_channel_info chinfo; + struct glink_channel *channel; + unsigned long flags; + + /* To wakeup any blocking writers */ + wake_up_all(&glink->tx_avail_notify); + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->lcids, lcid); + if (WARN(!channel, "close ack on unknown channel\n")) { + spin_unlock_irqrestore(&glink->idr_lock, flags); + return; + } + + idr_remove(&glink->lcids, channel->lcid); + channel->lcid = 0; + spin_unlock_irqrestore(&glink->idr_lock, flags); + + /* Decouple the potential rpdev from the channel */ + if (channel->rpdev) { + strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); + chinfo.src = RPMSG_ADDR_ANY; + chinfo.dst = RPMSG_ADDR_ANY; + + rpmsg_unregister_device(glink->dev, &chinfo); + } + channel->rpdev = NULL; + + kref_put(&channel->refcount, qcom_glink_channel_release); +} + +static void qcom_glink_work(struct work_struct *work) +{ + struct qcom_glink *glink = container_of(work, struct qcom_glink, + rx_work); + struct glink_defer_cmd *dcmd; + struct glink_msg *msg; + unsigned long flags; + unsigned int param1; + unsigned int param2; + unsigned int cmd; + + for (;;) { + spin_lock_irqsave(&glink->rx_lock, flags); + if (list_empty(&glink->rx_queue)) { + spin_unlock_irqrestore(&glink->rx_lock, flags); + break; + } + dcmd = list_first_entry(&glink->rx_queue, + struct glink_defer_cmd, node); + list_del(&dcmd->node); + spin_unlock_irqrestore(&glink->rx_lock, flags); + + msg = &dcmd->msg; + cmd = le16_to_cpu(msg->cmd); + param1 = le16_to_cpu(msg->param1); + param2 = le32_to_cpu(msg->param2); + + switch (cmd) { + case GLINK_CMD_VERSION: + qcom_glink_receive_version(glink, param1, param2); + break; + case GLINK_CMD_VERSION_ACK: + qcom_glink_receive_version_ack(glink, param1, param2); + break; + case GLINK_CMD_OPEN: + qcom_glink_rx_open(glink, param1, msg->data); + break; + case GLINK_CMD_CLOSE: + qcom_glink_rx_close(glink, param1); + break; + case GLINK_CMD_CLOSE_ACK: + qcom_glink_rx_close_ack(glink, param1); + break; + case GLINK_CMD_RX_INTENT_REQ: + qcom_glink_handle_intent_req(glink, param1, param2); + break; + default: + WARN(1, "Unknown defer object %d\n", cmd); + break; + } + + kfree(dcmd); + } +} + +static void qcom_glink_cancel_rx_work(struct qcom_glink *glink) +{ + struct glink_defer_cmd *dcmd; + struct glink_defer_cmd *tmp; + + /* cancel any pending deferred rx_work */ + cancel_work_sync(&glink->rx_work); + + list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node) + kfree(dcmd); +} + +static ssize_t rpmsg_name_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret = 0; + const char *name; + + ret = of_property_read_string(dev->of_node, "label", &name); + if (ret < 0) + name = dev->of_node->name; + + return sysfs_emit(buf, "%s\n", name); +} +static DEVICE_ATTR_RO(rpmsg_name); + +static struct attribute *qcom_glink_attrs[] = { + &dev_attr_rpmsg_name.attr, + NULL +}; +ATTRIBUTE_GROUPS(qcom_glink); + +static void qcom_glink_device_release(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct glink_channel *channel = to_glink_channel(rpdev->ept); + + /* Release qcom_glink_alloc_channel() reference */ + kref_put(&channel->refcount, qcom_glink_channel_release); + kfree(rpdev->driver_override); + kfree(rpdev); +} + +static int qcom_glink_create_chrdev(struct qcom_glink *glink) +{ + struct rpmsg_device *rpdev; + struct glink_channel *channel; + + rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); + if (!rpdev) + return -ENOMEM; + + channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev"); + if (IS_ERR(channel)) { + kfree(rpdev); + return PTR_ERR(channel); + } + channel->rpdev = rpdev; + + rpdev->ept = &channel->ept; + rpdev->ops = &glink_device_ops; + rpdev->dev.parent = glink->dev; + rpdev->dev.release = qcom_glink_device_release; + + return rpmsg_ctrldev_register_device(rpdev); +} + +struct qcom_glink *qcom_glink_native_probe(struct device *dev, + unsigned long features, + struct qcom_glink_pipe *rx, + struct qcom_glink_pipe *tx, + bool intentless) +{ + int ret; + struct qcom_glink *glink; + + glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); + if (!glink) + return ERR_PTR(-ENOMEM); + + glink->dev = dev; + glink->tx_pipe = tx; + glink->rx_pipe = rx; + + glink->features = features; + glink->intentless = intentless; + + spin_lock_init(&glink->tx_lock); + spin_lock_init(&glink->rx_lock); + INIT_LIST_HEAD(&glink->rx_queue); + INIT_WORK(&glink->rx_work, qcom_glink_work); + init_waitqueue_head(&glink->tx_avail_notify); + + spin_lock_init(&glink->idr_lock); + idr_init(&glink->lcids); + idr_init(&glink->rcids); + + glink->dev->groups = qcom_glink_groups; + + ret = device_add_groups(dev, qcom_glink_groups); + if (ret) + dev_err(dev, "failed to add groups\n"); + + ret = qcom_glink_send_version(glink); + if (ret) + return ERR_PTR(ret); + + ret = qcom_glink_create_chrdev(glink); + if (ret) + dev_err(glink->dev, "failed to register chrdev\n"); + + return glink; +} +EXPORT_SYMBOL_GPL(qcom_glink_native_probe); + +static int qcom_glink_remove_device(struct device *dev, void *data) +{ + device_unregister(dev); + + return 0; +} + +void qcom_glink_native_remove(struct qcom_glink *glink) +{ + struct glink_channel *channel; + unsigned long flags; + int cid; + int ret; + + qcom_glink_cancel_rx_work(glink); + + /* Fail all attempts at sending messages */ + spin_lock_irqsave(&glink->tx_lock, flags); + glink->abort_tx = true; + wake_up_all(&glink->tx_avail_notify); + spin_unlock_irqrestore(&glink->tx_lock, flags); + + /* Abort any senders waiting for intent requests */ + spin_lock_irqsave(&glink->idr_lock, flags); + idr_for_each_entry(&glink->lcids, channel, cid) + qcom_glink_intent_req_abort(channel); + spin_unlock_irqrestore(&glink->idr_lock, flags); + + ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device); + if (ret) + dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); + + /* Release any defunct local channels, waiting for close-ack */ + idr_for_each_entry(&glink->lcids, channel, cid) + kref_put(&channel->refcount, qcom_glink_channel_release); + + /* Release any defunct local channels, waiting for close-req */ + idr_for_each_entry(&glink->rcids, channel, cid) + kref_put(&channel->refcount, qcom_glink_channel_release); + + idr_destroy(&glink->lcids); + idr_destroy(&glink->rcids); +} +EXPORT_SYMBOL_GPL(qcom_glink_native_remove); + +MODULE_DESCRIPTION("Qualcomm GLINK driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/qcom_glink_native.h b/drivers/rpmsg/qcom_glink_native.h new file mode 100644 index 0000000000..8dbec24de2 --- /dev/null +++ b/drivers/rpmsg/qcom_glink_native.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2016-2017, Linaro Ltd + */ + +#ifndef __QCOM_GLINK_NATIVE_H__ +#define __QCOM_GLINK_NATIVE_H__ + +#include <linux/types.h> + +#define GLINK_FEATURE_INTENT_REUSE BIT(0) +#define GLINK_FEATURE_MIGRATION BIT(1) +#define GLINK_FEATURE_TRACER_PKT BIT(2) + +struct qcom_glink_pipe { + size_t length; + + size_t (*avail)(struct qcom_glink_pipe *glink_pipe); + + void (*peek)(struct qcom_glink_pipe *glink_pipe, void *data, + unsigned int offset, size_t count); + void (*advance)(struct qcom_glink_pipe *glink_pipe, size_t count); + + void (*write)(struct qcom_glink_pipe *glink_pipe, + const void *hdr, size_t hlen, + const void *data, size_t dlen); + void (*kick)(struct qcom_glink_pipe *glink_pipe); +}; + +struct device; +struct qcom_glink; + +struct qcom_glink *qcom_glink_native_probe(struct device *dev, + unsigned long features, + struct qcom_glink_pipe *rx, + struct qcom_glink_pipe *tx, + bool intentless); +void qcom_glink_native_remove(struct qcom_glink *glink); +void qcom_glink_native_rx(struct qcom_glink *glink); + +#endif diff --git a/drivers/rpmsg/qcom_glink_rpm.c b/drivers/rpmsg/qcom_glink_rpm.c new file mode 100644 index 0000000000..357272d706 --- /dev/null +++ b/drivers/rpmsg/qcom_glink_rpm.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016-2017, Linaro Ltd + */ + +#include <linux/idr.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/list.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/rpmsg.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include <linux/mailbox_client.h> + +#include "rpmsg_internal.h" +#include "qcom_glink_native.h" + +#define RPM_TOC_SIZE 256 +#define RPM_TOC_MAGIC 0x67727430 /* grt0 */ +#define RPM_TOC_MAX_ENTRIES ((RPM_TOC_SIZE - sizeof(struct rpm_toc)) / \ + sizeof(struct rpm_toc_entry)) + +#define RPM_TX_FIFO_ID 0x61703272 /* ap2r */ +#define RPM_RX_FIFO_ID 0x72326170 /* r2ap */ + +#define to_rpm_pipe(p) container_of(p, struct glink_rpm_pipe, native) + +struct rpm_toc_entry { + __le32 id; + __le32 offset; + __le32 size; +} __packed; + +struct rpm_toc { + __le32 magic; + __le32 count; + + struct rpm_toc_entry entries[]; +} __packed; + +struct glink_rpm_pipe { + struct qcom_glink_pipe native; + + void __iomem *tail; + void __iomem *head; + + void __iomem *fifo; +}; + +struct glink_rpm { + struct qcom_glink *glink; + + int irq; + + struct mbox_client mbox_client; + struct mbox_chan *mbox_chan; + + struct glink_rpm_pipe rx_pipe; + struct glink_rpm_pipe tx_pipe; +}; + +static size_t glink_rpm_rx_avail(struct qcom_glink_pipe *glink_pipe) +{ + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); + unsigned int head; + unsigned int tail; + + head = readl(pipe->head); + tail = readl(pipe->tail); + + if (head < tail) + return pipe->native.length - tail + head; + else + return head - tail; +} + +static void glink_rpm_rx_peek(struct qcom_glink_pipe *glink_pipe, + void *data, unsigned int offset, size_t count) +{ + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); + unsigned int tail; + size_t len; + + tail = readl(pipe->tail); + tail += offset; + if (tail >= pipe->native.length) + tail -= pipe->native.length; + + len = min_t(size_t, count, pipe->native.length - tail); + if (len) { + __ioread32_copy(data, pipe->fifo + tail, + len / sizeof(u32)); + } + + if (len != count) { + __ioread32_copy(data + len, pipe->fifo, + (count - len) / sizeof(u32)); + } +} + +static void glink_rpm_rx_advance(struct qcom_glink_pipe *glink_pipe, + size_t count) +{ + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); + unsigned int tail; + + tail = readl(pipe->tail); + + tail += count; + if (tail >= pipe->native.length) + tail -= pipe->native.length; + + writel(tail, pipe->tail); +} + +static size_t glink_rpm_tx_avail(struct qcom_glink_pipe *glink_pipe) +{ + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); + unsigned int head; + unsigned int tail; + + head = readl(pipe->head); + tail = readl(pipe->tail); + + if (tail <= head) + return pipe->native.length - head + tail; + else + return tail - head; +} + +static unsigned int glink_rpm_tx_write_one(struct glink_rpm_pipe *pipe, + unsigned int head, + const void *data, size_t count) +{ + size_t len; + + len = min_t(size_t, count, pipe->native.length - head); + if (len) { + __iowrite32_copy(pipe->fifo + head, data, + len / sizeof(u32)); + } + + if (len != count) { + __iowrite32_copy(pipe->fifo, data + len, + (count - len) / sizeof(u32)); + } + + head += count; + if (head >= pipe->native.length) + head -= pipe->native.length; + + return head; +} + +static void glink_rpm_tx_write(struct qcom_glink_pipe *glink_pipe, + const void *hdr, size_t hlen, + const void *data, size_t dlen) +{ + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); + size_t tlen = hlen + dlen; + size_t aligned_dlen; + unsigned int head; + char padding[8] = {0}; + size_t pad; + + /* Header length comes from glink native and is always 4 byte aligned */ + if (WARN(hlen % 4, "Glink Header length must be 4 bytes aligned\n")) + return; + + /* + * Move the unaligned tail of the message to the padding chunk, to + * ensure word aligned accesses + */ + aligned_dlen = ALIGN_DOWN(dlen, 4); + if (aligned_dlen != dlen) + memcpy(padding, data + aligned_dlen, dlen - aligned_dlen); + + head = readl(pipe->head); + head = glink_rpm_tx_write_one(pipe, head, hdr, hlen); + head = glink_rpm_tx_write_one(pipe, head, data, aligned_dlen); + + pad = ALIGN(tlen, 8) - ALIGN_DOWN(tlen, 4); + if (pad) + head = glink_rpm_tx_write_one(pipe, head, padding, pad); + writel(head, pipe->head); +} + +static void glink_rpm_tx_kick(struct qcom_glink_pipe *glink_pipe) +{ + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); + struct glink_rpm *rpm = container_of(pipe, struct glink_rpm, tx_pipe); + + mbox_send_message(rpm->mbox_chan, NULL); + mbox_client_txdone(rpm->mbox_chan, 0); +} + +static irqreturn_t qcom_glink_rpm_intr(int irq, void *data) +{ + struct glink_rpm *rpm = data; + + qcom_glink_native_rx(rpm->glink); + + return IRQ_HANDLED; +} + +static int glink_rpm_parse_toc(struct device *dev, + void __iomem *msg_ram, + size_t msg_ram_size, + struct glink_rpm_pipe *rx, + struct glink_rpm_pipe *tx) +{ + struct rpm_toc *toc; + int num_entries; + unsigned int id; + size_t offset; + size_t size; + void *buf; + int i; + + buf = kzalloc(RPM_TOC_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + __ioread32_copy(buf, msg_ram + msg_ram_size - RPM_TOC_SIZE, + RPM_TOC_SIZE / sizeof(u32)); + + toc = buf; + + if (le32_to_cpu(toc->magic) != RPM_TOC_MAGIC) { + dev_err(dev, "RPM TOC has invalid magic\n"); + goto err_inval; + } + + num_entries = le32_to_cpu(toc->count); + if (num_entries > RPM_TOC_MAX_ENTRIES) { + dev_err(dev, "Invalid number of toc entries\n"); + goto err_inval; + } + + for (i = 0; i < num_entries; i++) { + id = le32_to_cpu(toc->entries[i].id); + offset = le32_to_cpu(toc->entries[i].offset); + size = le32_to_cpu(toc->entries[i].size); + + if (offset > msg_ram_size || offset + size > msg_ram_size) { + dev_err(dev, "TOC entry with invalid size\n"); + continue; + } + + switch (id) { + case RPM_RX_FIFO_ID: + rx->native.length = size; + + rx->tail = msg_ram + offset; + rx->head = msg_ram + offset + sizeof(u32); + rx->fifo = msg_ram + offset + 2 * sizeof(u32); + break; + case RPM_TX_FIFO_ID: + tx->native.length = size; + + tx->tail = msg_ram + offset; + tx->head = msg_ram + offset + sizeof(u32); + tx->fifo = msg_ram + offset + 2 * sizeof(u32); + break; + } + } + + if (!rx->fifo || !tx->fifo) { + dev_err(dev, "Unable to find rx and tx descriptors\n"); + goto err_inval; + } + + kfree(buf); + return 0; + +err_inval: + kfree(buf); + return -EINVAL; +} + +static int glink_rpm_probe(struct platform_device *pdev) +{ + struct qcom_glink *glink; + struct glink_rpm *rpm; + struct device_node *np; + void __iomem *msg_ram; + size_t msg_ram_size; + struct device *dev = &pdev->dev; + struct resource r; + int ret; + + rpm = devm_kzalloc(&pdev->dev, sizeof(*rpm), GFP_KERNEL); + if (!rpm) + return -ENOMEM; + + np = of_parse_phandle(dev->of_node, "qcom,rpm-msg-ram", 0); + ret = of_address_to_resource(np, 0, &r); + of_node_put(np); + if (ret) + return ret; + + msg_ram = devm_ioremap(dev, r.start, resource_size(&r)); + msg_ram_size = resource_size(&r); + if (!msg_ram) + return -ENOMEM; + + ret = glink_rpm_parse_toc(dev, msg_ram, msg_ram_size, + &rpm->rx_pipe, &rpm->tx_pipe); + if (ret) + return ret; + + rpm->irq = of_irq_get(dev->of_node, 0); + ret = devm_request_irq(dev, rpm->irq, qcom_glink_rpm_intr, + IRQF_NO_SUSPEND | IRQF_NO_AUTOEN, + "glink-rpm", rpm); + if (ret) { + dev_err(dev, "failed to request IRQ\n"); + return ret; + } + + rpm->mbox_client.dev = dev; + rpm->mbox_client.knows_txdone = true; + rpm->mbox_chan = mbox_request_channel(&rpm->mbox_client, 0); + if (IS_ERR(rpm->mbox_chan)) + return dev_err_probe(dev, PTR_ERR(rpm->mbox_chan), "failed to acquire IPC channel\n"); + + /* Pipe specific accessors */ + rpm->rx_pipe.native.avail = glink_rpm_rx_avail; + rpm->rx_pipe.native.peek = glink_rpm_rx_peek; + rpm->rx_pipe.native.advance = glink_rpm_rx_advance; + rpm->tx_pipe.native.avail = glink_rpm_tx_avail; + rpm->tx_pipe.native.write = glink_rpm_tx_write; + rpm->tx_pipe.native.kick = glink_rpm_tx_kick; + + writel(0, rpm->tx_pipe.head); + writel(0, rpm->rx_pipe.tail); + + glink = qcom_glink_native_probe(dev, + 0, + &rpm->rx_pipe.native, + &rpm->tx_pipe.native, + true); + if (IS_ERR(glink)) { + mbox_free_channel(rpm->mbox_chan); + return PTR_ERR(glink); + } + + rpm->glink = glink; + + platform_set_drvdata(pdev, rpm); + + enable_irq(rpm->irq); + + return 0; +} + +static void glink_rpm_remove(struct platform_device *pdev) +{ + struct glink_rpm *rpm = platform_get_drvdata(pdev); + struct qcom_glink *glink = rpm->glink; + + disable_irq(rpm->irq); + + qcom_glink_native_remove(glink); + + mbox_free_channel(rpm->mbox_chan); +} + +static const struct of_device_id glink_rpm_of_match[] = { + { .compatible = "qcom,glink-rpm" }, + {} +}; +MODULE_DEVICE_TABLE(of, glink_rpm_of_match); + +static struct platform_driver glink_rpm_driver = { + .probe = glink_rpm_probe, + .remove_new = glink_rpm_remove, + .driver = { + .name = "qcom_glink_rpm", + .of_match_table = glink_rpm_of_match, + }, +}; + +static int __init glink_rpm_init(void) +{ + return platform_driver_register(&glink_rpm_driver); +} +subsys_initcall(glink_rpm_init); + +static void __exit glink_rpm_exit(void) +{ + platform_driver_unregister(&glink_rpm_driver); +} +module_exit(glink_rpm_exit); + +MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@linaro.org>"); +MODULE_DESCRIPTION("Qualcomm GLINK RPM driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/qcom_glink_smem.c b/drivers/rpmsg/qcom_glink_smem.c new file mode 100644 index 0000000000..7a982c60a8 --- /dev/null +++ b/drivers/rpmsg/qcom_glink_smem.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016, Linaro Ltd + */ + +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/mailbox_client.h> +#include <linux/mfd/syscon.h> +#include <linux/slab.h> +#include <linux/rpmsg.h> +#include <linux/idr.h> +#include <linux/circ_buf.h> +#include <linux/soc/qcom/smem.h> +#include <linux/sizes.h> +#include <linux/delay.h> +#include <linux/regmap.h> +#include <linux/workqueue.h> +#include <linux/list.h> + +#include <linux/rpmsg/qcom_glink.h> + +#include "qcom_glink_native.h" + +#define FIFO_FULL_RESERVE 8 +#define FIFO_ALIGNMENT 8 +#define TX_BLOCKED_CMD_RESERVE 8 /* size of struct read_notif_request */ + +#define SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR 478 +#define SMEM_GLINK_NATIVE_XPRT_FIFO_0 479 +#define SMEM_GLINK_NATIVE_XPRT_FIFO_1 480 + +struct qcom_glink_smem { + struct device dev; + + int irq; + struct qcom_glink *glink; + + struct mbox_client mbox_client; + struct mbox_chan *mbox_chan; + + u32 remote_pid; +}; + +struct glink_smem_pipe { + struct qcom_glink_pipe native; + + __le32 *tail; + __le32 *head; + + void *fifo; + + struct qcom_glink_smem *smem; +}; + +#define to_smem_pipe(p) container_of(p, struct glink_smem_pipe, native) + +static size_t glink_smem_rx_avail(struct qcom_glink_pipe *np) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(np); + struct qcom_glink_smem *smem = pipe->smem; + size_t len; + void *fifo; + u32 head; + u32 tail; + + if (!pipe->fifo) { + fifo = qcom_smem_get(smem->remote_pid, + SMEM_GLINK_NATIVE_XPRT_FIFO_1, &len); + if (IS_ERR(fifo)) { + pr_err("failed to acquire RX fifo handle: %ld\n", + PTR_ERR(fifo)); + return 0; + } + + pipe->fifo = fifo; + pipe->native.length = len; + } + + head = le32_to_cpu(*pipe->head); + tail = le32_to_cpu(*pipe->tail); + + if (head < tail) + return pipe->native.length - tail + head; + else + return head - tail; +} + +static void glink_smem_rx_peek(struct qcom_glink_pipe *np, + void *data, unsigned int offset, size_t count) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(np); + size_t len; + u32 tail; + + tail = le32_to_cpu(*pipe->tail); + tail += offset; + if (tail >= pipe->native.length) + tail -= pipe->native.length; + + len = min_t(size_t, count, pipe->native.length - tail); + if (len) + memcpy_fromio(data, pipe->fifo + tail, len); + + if (len != count) + memcpy_fromio(data + len, pipe->fifo, (count - len)); +} + +static void glink_smem_rx_advance(struct qcom_glink_pipe *np, + size_t count) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(np); + u32 tail; + + tail = le32_to_cpu(*pipe->tail); + + tail += count; + if (tail >= pipe->native.length) + tail -= pipe->native.length; + + *pipe->tail = cpu_to_le32(tail); +} + +static size_t glink_smem_tx_avail(struct qcom_glink_pipe *np) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(np); + u32 head; + u32 tail; + u32 avail; + + head = le32_to_cpu(*pipe->head); + tail = le32_to_cpu(*pipe->tail); + + if (tail <= head) + avail = pipe->native.length - head + tail; + else + avail = tail - head; + + if (avail < (FIFO_FULL_RESERVE + TX_BLOCKED_CMD_RESERVE)) + avail = 0; + else + avail -= FIFO_FULL_RESERVE + TX_BLOCKED_CMD_RESERVE; + + return avail; +} + +static unsigned int glink_smem_tx_write_one(struct glink_smem_pipe *pipe, + unsigned int head, + const void *data, size_t count) +{ + size_t len; + + len = min_t(size_t, count, pipe->native.length - head); + if (len) + memcpy(pipe->fifo + head, data, len); + + if (len != count) + memcpy(pipe->fifo, data + len, count - len); + + head += count; + if (head >= pipe->native.length) + head -= pipe->native.length; + + return head; +} + +static void glink_smem_tx_write(struct qcom_glink_pipe *glink_pipe, + const void *hdr, size_t hlen, + const void *data, size_t dlen) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(glink_pipe); + unsigned int head; + + head = le32_to_cpu(*pipe->head); + + head = glink_smem_tx_write_one(pipe, head, hdr, hlen); + head = glink_smem_tx_write_one(pipe, head, data, dlen); + + /* Ensure head is always aligned to 8 bytes */ + head = ALIGN(head, 8); + if (head >= pipe->native.length) + head -= pipe->native.length; + + /* Ensure ordering of fifo and head update */ + wmb(); + + *pipe->head = cpu_to_le32(head); +} + +static void glink_smem_tx_kick(struct qcom_glink_pipe *glink_pipe) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(glink_pipe); + struct qcom_glink_smem *smem = pipe->smem; + + mbox_send_message(smem->mbox_chan, NULL); + mbox_client_txdone(smem->mbox_chan, 0); +} + +static irqreturn_t qcom_glink_smem_intr(int irq, void *data) +{ + struct qcom_glink_smem *smem = data; + + qcom_glink_native_rx(smem->glink); + + return IRQ_HANDLED; +} + +static void qcom_glink_smem_release(struct device *dev) +{ + struct qcom_glink_smem *smem = container_of(dev, struct qcom_glink_smem, dev); + + kfree(smem); +} + +struct qcom_glink_smem *qcom_glink_smem_register(struct device *parent, + struct device_node *node) +{ + struct glink_smem_pipe *rx_pipe; + struct glink_smem_pipe *tx_pipe; + struct qcom_glink_smem *smem; + struct qcom_glink *glink; + struct device *dev; + u32 remote_pid; + __le32 *descs; + size_t size; + int ret; + + smem = kzalloc(sizeof(*smem), GFP_KERNEL); + if (!smem) + return ERR_PTR(-ENOMEM); + + dev = &smem->dev; + + dev->parent = parent; + dev->of_node = node; + dev->release = qcom_glink_smem_release; + dev_set_name(dev, "%s:%pOFn", dev_name(parent->parent), node); + ret = device_register(dev); + if (ret) { + pr_err("failed to register glink edge\n"); + put_device(dev); + return ERR_PTR(ret); + } + + ret = of_property_read_u32(dev->of_node, "qcom,remote-pid", + &remote_pid); + if (ret) { + dev_err(dev, "failed to parse qcom,remote-pid\n"); + goto err_put_dev; + } + + smem->remote_pid = remote_pid; + + rx_pipe = devm_kzalloc(dev, sizeof(*rx_pipe), GFP_KERNEL); + tx_pipe = devm_kzalloc(dev, sizeof(*tx_pipe), GFP_KERNEL); + if (!rx_pipe || !tx_pipe) { + ret = -ENOMEM; + goto err_put_dev; + } + + ret = qcom_smem_alloc(remote_pid, + SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR, 32); + if (ret && ret != -EEXIST) { + dev_err(dev, "failed to allocate glink descriptors\n"); + goto err_put_dev; + } + + descs = qcom_smem_get(remote_pid, + SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR, &size); + if (IS_ERR(descs)) { + dev_err(dev, "failed to acquire xprt descriptor\n"); + ret = PTR_ERR(descs); + goto err_put_dev; + } + + if (size != 32) { + dev_err(dev, "glink descriptor of invalid size\n"); + ret = -EINVAL; + goto err_put_dev; + } + + tx_pipe->tail = &descs[0]; + tx_pipe->head = &descs[1]; + rx_pipe->tail = &descs[2]; + rx_pipe->head = &descs[3]; + + ret = qcom_smem_alloc(remote_pid, SMEM_GLINK_NATIVE_XPRT_FIFO_0, + SZ_16K); + if (ret && ret != -EEXIST) { + dev_err(dev, "failed to allocate TX fifo\n"); + goto err_put_dev; + } + + tx_pipe->fifo = qcom_smem_get(remote_pid, SMEM_GLINK_NATIVE_XPRT_FIFO_0, + &tx_pipe->native.length); + if (IS_ERR(tx_pipe->fifo)) { + dev_err(dev, "failed to acquire TX fifo\n"); + ret = PTR_ERR(tx_pipe->fifo); + goto err_put_dev; + } + + smem->irq = of_irq_get(smem->dev.of_node, 0); + ret = devm_request_irq(&smem->dev, smem->irq, qcom_glink_smem_intr, + IRQF_NO_SUSPEND | IRQF_NO_AUTOEN, + "glink-smem", smem); + if (ret) { + dev_err(&smem->dev, "failed to request IRQ\n"); + goto err_put_dev; + } + + smem->mbox_client.dev = &smem->dev; + smem->mbox_client.knows_txdone = true; + smem->mbox_chan = mbox_request_channel(&smem->mbox_client, 0); + if (IS_ERR(smem->mbox_chan)) { + ret = dev_err_probe(&smem->dev, PTR_ERR(smem->mbox_chan), + "failed to acquire IPC channel\n"); + goto err_put_dev; + } + + rx_pipe->smem = smem; + rx_pipe->native.avail = glink_smem_rx_avail; + rx_pipe->native.peek = glink_smem_rx_peek; + rx_pipe->native.advance = glink_smem_rx_advance; + + tx_pipe->smem = smem; + tx_pipe->native.avail = glink_smem_tx_avail; + tx_pipe->native.write = glink_smem_tx_write; + tx_pipe->native.kick = glink_smem_tx_kick; + + *rx_pipe->tail = 0; + *tx_pipe->head = 0; + + glink = qcom_glink_native_probe(dev, + GLINK_FEATURE_INTENT_REUSE, + &rx_pipe->native, &tx_pipe->native, + false); + if (IS_ERR(glink)) { + ret = PTR_ERR(glink); + goto err_free_mbox; + } + + smem->glink = glink; + + enable_irq(smem->irq); + + return smem; + +err_free_mbox: + mbox_free_channel(smem->mbox_chan); + +err_put_dev: + device_unregister(dev); + + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(qcom_glink_smem_register); + +void qcom_glink_smem_unregister(struct qcom_glink_smem *smem) +{ + struct qcom_glink *glink = smem->glink; + + disable_irq(smem->irq); + + qcom_glink_native_remove(glink); + + mbox_free_channel(smem->mbox_chan); + device_unregister(&smem->dev); +} +EXPORT_SYMBOL_GPL(qcom_glink_smem_unregister); + +MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@linaro.org>"); +MODULE_DESCRIPTION("Qualcomm GLINK SMEM driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/qcom_glink_ssr.c b/drivers/rpmsg/qcom_glink_ssr.c new file mode 100644 index 0000000000..39ffa384c9 --- /dev/null +++ b/drivers/rpmsg/qcom_glink_ssr.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014-2017, The Linux Foundation. All rights reserved. + * Copyright (c) 2017, Linaro Ltd. + */ + +#include <linux/completion.h> +#include <linux/module.h> +#include <linux/notifier.h> +#include <linux/rpmsg.h> +#include <linux/rpmsg/qcom_glink.h> +#include <linux/remoteproc/qcom_rproc.h> + +/** + * struct do_cleanup_msg - The data structure for an SSR do_cleanup message + * @version: The G-Link SSR protocol version + * @command: The G-Link SSR command - do_cleanup + * @seq_num: Sequence number + * @name_len: Length of the name of the subsystem being restarted + * @name: G-Link edge name of the subsystem being restarted + */ +struct do_cleanup_msg { + __le32 version; + __le32 command; + __le32 seq_num; + __le32 name_len; + char name[32]; +}; + +/** + * struct cleanup_done_msg - The data structure for an SSR cleanup_done message + * @version: The G-Link SSR protocol version + * @response: The G-Link SSR response to a do_cleanup command, cleanup_done + * @seq_num: Sequence number + */ +struct cleanup_done_msg { + __le32 version; + __le32 response; + __le32 seq_num; +}; + +/* + * G-Link SSR protocol commands + */ +#define GLINK_SSR_DO_CLEANUP 0 +#define GLINK_SSR_CLEANUP_DONE 1 + +struct glink_ssr { + struct device *dev; + struct rpmsg_endpoint *ept; + + struct notifier_block nb; + + u32 seq_num; + struct completion completion; +}; + +/* Notifier list for all registered glink_ssr instances */ +static BLOCKING_NOTIFIER_HEAD(ssr_notifiers); + +/** + * qcom_glink_ssr_notify() - notify GLINK SSR about stopped remoteproc + * @ssr_name: name of the remoteproc that has been stopped + */ +void qcom_glink_ssr_notify(const char *ssr_name) +{ + blocking_notifier_call_chain(&ssr_notifiers, 0, (void *)ssr_name); +} +EXPORT_SYMBOL_GPL(qcom_glink_ssr_notify); + +static int qcom_glink_ssr_callback(struct rpmsg_device *rpdev, + void *data, int len, void *priv, u32 addr) +{ + struct cleanup_done_msg *msg = data; + struct glink_ssr *ssr = dev_get_drvdata(&rpdev->dev); + + if (len < sizeof(*msg)) { + dev_err(ssr->dev, "message too short\n"); + return -EINVAL; + } + + if (le32_to_cpu(msg->version) != 0) + return -EINVAL; + + if (le32_to_cpu(msg->response) != GLINK_SSR_CLEANUP_DONE) + return 0; + + if (le32_to_cpu(msg->seq_num) != ssr->seq_num) { + dev_err(ssr->dev, "invalid sequence number of response\n"); + return -EINVAL; + } + + complete(&ssr->completion); + + return 0; +} + +static int qcom_glink_ssr_notifier_call(struct notifier_block *nb, + unsigned long event, + void *data) +{ + struct glink_ssr *ssr = container_of(nb, struct glink_ssr, nb); + struct do_cleanup_msg msg; + char *ssr_name = data; + int ret; + + ssr->seq_num++; + reinit_completion(&ssr->completion); + + memset(&msg, 0, sizeof(msg)); + msg.command = cpu_to_le32(GLINK_SSR_DO_CLEANUP); + msg.seq_num = cpu_to_le32(ssr->seq_num); + msg.name_len = cpu_to_le32(strlen(ssr_name)); + strscpy(msg.name, ssr_name, sizeof(msg.name)); + + ret = rpmsg_send(ssr->ept, &msg, sizeof(msg)); + if (ret < 0) + dev_err(ssr->dev, "failed to send cleanup message\n"); + + ret = wait_for_completion_timeout(&ssr->completion, HZ); + if (!ret) + dev_err(ssr->dev, "timeout waiting for cleanup done message\n"); + + return NOTIFY_DONE; +} + +static int qcom_glink_ssr_probe(struct rpmsg_device *rpdev) +{ + struct glink_ssr *ssr; + + ssr = devm_kzalloc(&rpdev->dev, sizeof(*ssr), GFP_KERNEL); + if (!ssr) + return -ENOMEM; + + init_completion(&ssr->completion); + + ssr->dev = &rpdev->dev; + ssr->ept = rpdev->ept; + ssr->nb.notifier_call = qcom_glink_ssr_notifier_call; + + dev_set_drvdata(&rpdev->dev, ssr); + + return blocking_notifier_chain_register(&ssr_notifiers, &ssr->nb); +} + +static void qcom_glink_ssr_remove(struct rpmsg_device *rpdev) +{ + struct glink_ssr *ssr = dev_get_drvdata(&rpdev->dev); + + blocking_notifier_chain_unregister(&ssr_notifiers, &ssr->nb); +} + +static const struct rpmsg_device_id qcom_glink_ssr_match[] = { + { "glink_ssr" }, + {} +}; + +static struct rpmsg_driver qcom_glink_ssr_driver = { + .probe = qcom_glink_ssr_probe, + .remove = qcom_glink_ssr_remove, + .callback = qcom_glink_ssr_callback, + .id_table = qcom_glink_ssr_match, + .drv = { + .name = "qcom_glink_ssr", + }, +}; +module_rpmsg_driver(qcom_glink_ssr_driver); diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c new file mode 100644 index 0000000000..43f601c84b --- /dev/null +++ b/drivers/rpmsg/qcom_smd.c @@ -0,0 +1,1620 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2015, Sony Mobile Communications AB. + * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. + */ + +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/mailbox_client.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of_irq.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/sched.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/soc/qcom/smem.h> +#include <linux/wait.h> +#include <linux/rpmsg.h> +#include <linux/rpmsg/qcom_smd.h> + +#include "rpmsg_internal.h" + +/* + * The Qualcomm Shared Memory communication solution provides point-to-point + * channels for clients to send and receive streaming or packet based data. + * + * Each channel consists of a control item (channel info) and a ring buffer + * pair. The channel info carry information related to channel state, flow + * control and the offsets within the ring buffer. + * + * All allocated channels are listed in an allocation table, identifying the + * pair of items by name, type and remote processor. + * + * Upon creating a new channel the remote processor allocates channel info and + * ring buffer items from the smem heap and populate the allocation table. An + * interrupt is sent to the other end of the channel and a scan for new + * channels should be done. A channel never goes away, it will only change + * state. + * + * The remote processor signals it intent for bring up the communication + * channel by setting the state of its end of the channel to "opening" and + * sends out an interrupt. We detect this change and register a smd device to + * consume the channel. Upon finding a consumer we finish the handshake and the + * channel is up. + * + * Upon closing a channel, the remote processor will update the state of its + * end of the channel and signal us, we will then unregister any attached + * device and close our end of the channel. + * + * Devices attached to a channel can use the qcom_smd_send function to push + * data to the channel, this is done by copying the data into the tx ring + * buffer, updating the pointers in the channel info and signaling the remote + * processor. + * + * The remote processor does the equivalent when it transfer data and upon + * receiving the interrupt we check the channel info for new data and delivers + * this to the attached device. If the device is not ready to receive the data + * we leave it in the ring buffer for now. + */ + +struct smd_channel_info; +struct smd_channel_info_pair; +struct smd_channel_info_word; +struct smd_channel_info_word_pair; + +static const struct rpmsg_endpoint_ops qcom_smd_endpoint_ops; + +#define SMD_ALLOC_TBL_COUNT 2 +#define SMD_ALLOC_TBL_SIZE 64 + +/* + * This lists the various smem heap items relevant for the allocation table and + * smd channel entries. + */ +static const struct { + unsigned alloc_tbl_id; + unsigned info_base_id; + unsigned fifo_base_id; +} smem_items[SMD_ALLOC_TBL_COUNT] = { + { + .alloc_tbl_id = 13, + .info_base_id = 14, + .fifo_base_id = 338 + }, + { + .alloc_tbl_id = 266, + .info_base_id = 138, + .fifo_base_id = 202, + }, +}; + +/** + * struct qcom_smd_edge - representing a remote processor + * @dev: device associated with this edge + * @name: name of this edge + * @of_node: of_node handle for information related to this edge + * @edge_id: identifier of this edge + * @remote_pid: identifier of remote processor + * @irq: interrupt for signals on this edge + * @ipc_regmap: regmap handle holding the outgoing ipc register + * @ipc_offset: offset within @ipc_regmap of the register for ipc + * @ipc_bit: bit in the register at @ipc_offset of @ipc_regmap + * @mbox_client: mailbox client handle + * @mbox_chan: apcs ipc mailbox channel handle + * @channels: list of all channels detected on this edge + * @channels_lock: guard for modifications of @channels + * @allocated: array of bitmaps representing already allocated channels + * @smem_available: last available amount of smem triggering a channel scan + * @new_channel_event: wait queue for new channel events + * @scan_work: work item for discovering new channels + * @state_work: work item for edge state changes + */ +struct qcom_smd_edge { + struct device dev; + + const char *name; + + struct device_node *of_node; + unsigned edge_id; + unsigned remote_pid; + + int irq; + + struct regmap *ipc_regmap; + int ipc_offset; + int ipc_bit; + + struct mbox_client mbox_client; + struct mbox_chan *mbox_chan; + + struct list_head channels; + spinlock_t channels_lock; + + DECLARE_BITMAP(allocated[SMD_ALLOC_TBL_COUNT], SMD_ALLOC_TBL_SIZE); + + unsigned smem_available; + + wait_queue_head_t new_channel_event; + + struct work_struct scan_work; + struct work_struct state_work; +}; + +/* + * SMD channel states. + */ +enum smd_channel_state { + SMD_CHANNEL_CLOSED, + SMD_CHANNEL_OPENING, + SMD_CHANNEL_OPENED, + SMD_CHANNEL_FLUSHING, + SMD_CHANNEL_CLOSING, + SMD_CHANNEL_RESET, + SMD_CHANNEL_RESET_OPENING +}; + +struct qcom_smd_device { + struct rpmsg_device rpdev; + + struct qcom_smd_edge *edge; +}; + +struct qcom_smd_endpoint { + struct rpmsg_endpoint ept; + + struct qcom_smd_channel *qsch; +}; + +#define to_smd_device(r) container_of(r, struct qcom_smd_device, rpdev) +#define to_smd_edge(d) container_of(d, struct qcom_smd_edge, dev) +#define to_smd_endpoint(e) container_of(e, struct qcom_smd_endpoint, ept) + +/** + * struct qcom_smd_channel - smd channel struct + * @edge: qcom_smd_edge this channel is living on + * @qsept: reference to a associated smd endpoint + * @registered: flag to indicate if the channel is registered + * @name: name of the channel + * @state: local state of the channel + * @remote_state: remote state of the channel + * @state_change_event: state change event + * @info: byte aligned outgoing/incoming channel info + * @info_word: word aligned outgoing/incoming channel info + * @tx_lock: lock to make writes to the channel mutually exclusive + * @fblockread_event: wakeup event tied to tx fBLOCKREADINTR + * @tx_fifo: pointer to the outgoing ring buffer + * @rx_fifo: pointer to the incoming ring buffer + * @fifo_size: size of each ring buffer + * @bounce_buffer: bounce buffer for reading wrapped packets + * @cb: callback function registered for this channel + * @recv_lock: guard for rx info modifications and cb pointer + * @pkt_size: size of the currently handled packet + * @drvdata: driver private data + * @list: lite entry for @channels in qcom_smd_edge + */ +struct qcom_smd_channel { + struct qcom_smd_edge *edge; + + struct qcom_smd_endpoint *qsept; + bool registered; + + char *name; + enum smd_channel_state state; + enum smd_channel_state remote_state; + wait_queue_head_t state_change_event; + + struct smd_channel_info_pair *info; + struct smd_channel_info_word_pair *info_word; + + spinlock_t tx_lock; + wait_queue_head_t fblockread_event; + + void *tx_fifo; + void *rx_fifo; + int fifo_size; + + void *bounce_buffer; + + spinlock_t recv_lock; + + int pkt_size; + + void *drvdata; + + struct list_head list; +}; + +/* + * Format of the smd_info smem items, for byte aligned channels. + */ +struct smd_channel_info { + __le32 state; + u8 fDSR; + u8 fCTS; + u8 fCD; + u8 fRI; + u8 fHEAD; + u8 fTAIL; + u8 fSTATE; + u8 fBLOCKREADINTR; + __le32 tail; + __le32 head; +}; + +struct smd_channel_info_pair { + struct smd_channel_info tx; + struct smd_channel_info rx; +}; + +/* + * Format of the smd_info smem items, for word aligned channels. + */ +struct smd_channel_info_word { + __le32 state; + __le32 fDSR; + __le32 fCTS; + __le32 fCD; + __le32 fRI; + __le32 fHEAD; + __le32 fTAIL; + __le32 fSTATE; + __le32 fBLOCKREADINTR; + __le32 tail; + __le32 head; +}; + +struct smd_channel_info_word_pair { + struct smd_channel_info_word tx; + struct smd_channel_info_word rx; +}; + +#define GET_RX_CHANNEL_FLAG(channel, param) \ + ({ \ + BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u8)); \ + channel->info_word ? \ + le32_to_cpu(channel->info_word->rx.param) : \ + channel->info->rx.param; \ + }) + +#define GET_RX_CHANNEL_INFO(channel, param) \ + ({ \ + BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u32)); \ + le32_to_cpu(channel->info_word ? \ + channel->info_word->rx.param : \ + channel->info->rx.param); \ + }) + +#define SET_RX_CHANNEL_FLAG(channel, param, value) \ + ({ \ + BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u8)); \ + if (channel->info_word) \ + channel->info_word->rx.param = cpu_to_le32(value); \ + else \ + channel->info->rx.param = value; \ + }) + +#define SET_RX_CHANNEL_INFO(channel, param, value) \ + ({ \ + BUILD_BUG_ON(sizeof(channel->info->rx.param) != sizeof(u32)); \ + if (channel->info_word) \ + channel->info_word->rx.param = cpu_to_le32(value); \ + else \ + channel->info->rx.param = cpu_to_le32(value); \ + }) + +#define GET_TX_CHANNEL_FLAG(channel, param) \ + ({ \ + BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u8)); \ + channel->info_word ? \ + le32_to_cpu(channel->info_word->tx.param) : \ + channel->info->tx.param; \ + }) + +#define GET_TX_CHANNEL_INFO(channel, param) \ + ({ \ + BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u32)); \ + le32_to_cpu(channel->info_word ? \ + channel->info_word->tx.param : \ + channel->info->tx.param); \ + }) + +#define SET_TX_CHANNEL_FLAG(channel, param, value) \ + ({ \ + BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u8)); \ + if (channel->info_word) \ + channel->info_word->tx.param = cpu_to_le32(value); \ + else \ + channel->info->tx.param = value; \ + }) + +#define SET_TX_CHANNEL_INFO(channel, param, value) \ + ({ \ + BUILD_BUG_ON(sizeof(channel->info->tx.param) != sizeof(u32)); \ + if (channel->info_word) \ + channel->info_word->tx.param = cpu_to_le32(value); \ + else \ + channel->info->tx.param = cpu_to_le32(value); \ + }) + +/** + * struct qcom_smd_alloc_entry - channel allocation entry + * @name: channel name + * @cid: channel index + * @flags: channel flags and edge id + * @ref_count: reference count of the channel + */ +struct qcom_smd_alloc_entry { + u8 name[20]; + __le32 cid; + __le32 flags; + __le32 ref_count; +} __packed; + +#define SMD_CHANNEL_FLAGS_EDGE_MASK 0xff +#define SMD_CHANNEL_FLAGS_STREAM BIT(8) +#define SMD_CHANNEL_FLAGS_PACKET BIT(9) + +/* + * Each smd packet contains a 20 byte header, with the first 4 being the length + * of the packet. + */ +#define SMD_PACKET_HEADER_LEN 20 + +/* + * Signal the remote processor associated with 'channel'. + */ +static void qcom_smd_signal_channel(struct qcom_smd_channel *channel) +{ + struct qcom_smd_edge *edge = channel->edge; + + if (edge->mbox_chan) { + /* + * We can ignore a failing mbox_send_message() as the only + * possible cause is that the FIFO in the framework is full of + * other writes to the same bit. + */ + mbox_send_message(edge->mbox_chan, NULL); + mbox_client_txdone(edge->mbox_chan, 0); + } else { + regmap_write(edge->ipc_regmap, edge->ipc_offset, BIT(edge->ipc_bit)); + } +} + +/* + * Initialize the tx channel info + */ +static void qcom_smd_channel_reset(struct qcom_smd_channel *channel) +{ + SET_TX_CHANNEL_INFO(channel, state, SMD_CHANNEL_CLOSED); + SET_TX_CHANNEL_FLAG(channel, fDSR, 0); + SET_TX_CHANNEL_FLAG(channel, fCTS, 0); + SET_TX_CHANNEL_FLAG(channel, fCD, 0); + SET_TX_CHANNEL_FLAG(channel, fRI, 0); + SET_TX_CHANNEL_FLAG(channel, fHEAD, 0); + SET_TX_CHANNEL_FLAG(channel, fTAIL, 0); + SET_TX_CHANNEL_FLAG(channel, fSTATE, 1); + SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 1); + SET_TX_CHANNEL_INFO(channel, head, 0); + SET_RX_CHANNEL_INFO(channel, tail, 0); + + qcom_smd_signal_channel(channel); + + channel->state = SMD_CHANNEL_CLOSED; + channel->pkt_size = 0; +} + +/* + * Set the callback for a channel, with appropriate locking + */ +static void qcom_smd_channel_set_callback(struct qcom_smd_channel *channel, + rpmsg_rx_cb_t cb) +{ + struct rpmsg_endpoint *ept = &channel->qsept->ept; + unsigned long flags; + + spin_lock_irqsave(&channel->recv_lock, flags); + ept->cb = cb; + spin_unlock_irqrestore(&channel->recv_lock, flags); +}; + +/* + * Calculate the amount of data available in the rx fifo + */ +static size_t qcom_smd_channel_get_rx_avail(struct qcom_smd_channel *channel) +{ + unsigned head; + unsigned tail; + + head = GET_RX_CHANNEL_INFO(channel, head); + tail = GET_RX_CHANNEL_INFO(channel, tail); + + return (head - tail) & (channel->fifo_size - 1); +} + +/* + * Set tx channel state and inform the remote processor + */ +static void qcom_smd_channel_set_state(struct qcom_smd_channel *channel, + int state) +{ + struct qcom_smd_edge *edge = channel->edge; + bool is_open = state == SMD_CHANNEL_OPENED; + + if (channel->state == state) + return; + + dev_dbg(&edge->dev, "set_state(%s, %d)\n", channel->name, state); + + SET_TX_CHANNEL_FLAG(channel, fDSR, is_open); + SET_TX_CHANNEL_FLAG(channel, fCTS, is_open); + SET_TX_CHANNEL_FLAG(channel, fCD, is_open); + + SET_TX_CHANNEL_INFO(channel, state, state); + SET_TX_CHANNEL_FLAG(channel, fSTATE, 1); + + channel->state = state; + qcom_smd_signal_channel(channel); +} + +/* + * Copy count bytes of data using 32bit accesses, if that's required. + */ +static void smd_copy_to_fifo(void __iomem *dst, + const void *src, + size_t count, + bool word_aligned) +{ + if (word_aligned) { + __iowrite32_copy(dst, src, count / sizeof(u32)); + } else { + memcpy_toio(dst, src, count); + } +} + +/* + * Copy count bytes of data using 32bit accesses, if that is required. + */ +static void smd_copy_from_fifo(void *dst, + const void __iomem *src, + size_t count, + bool word_aligned) +{ + if (word_aligned) { + __ioread32_copy(dst, src, count / sizeof(u32)); + } else { + memcpy_fromio(dst, src, count); + } +} + +/* + * Read count bytes of data from the rx fifo into buf, but don't advance the + * tail. + */ +static size_t qcom_smd_channel_peek(struct qcom_smd_channel *channel, + void *buf, size_t count) +{ + bool word_aligned; + unsigned tail; + size_t len; + + word_aligned = channel->info_word; + tail = GET_RX_CHANNEL_INFO(channel, tail); + + len = min_t(size_t, count, channel->fifo_size - tail); + if (len) { + smd_copy_from_fifo(buf, + channel->rx_fifo + tail, + len, + word_aligned); + } + + if (len != count) { + smd_copy_from_fifo(buf + len, + channel->rx_fifo, + count - len, + word_aligned); + } + + return count; +} + +/* + * Advance the rx tail by count bytes. + */ +static void qcom_smd_channel_advance(struct qcom_smd_channel *channel, + size_t count) +{ + unsigned tail; + + tail = GET_RX_CHANNEL_INFO(channel, tail); + tail += count; + tail &= (channel->fifo_size - 1); + SET_RX_CHANNEL_INFO(channel, tail, tail); +} + +/* + * Read out a single packet from the rx fifo and deliver it to the device + */ +static int qcom_smd_channel_recv_single(struct qcom_smd_channel *channel) +{ + struct rpmsg_endpoint *ept = &channel->qsept->ept; + unsigned tail; + size_t len; + void *ptr; + int ret; + + tail = GET_RX_CHANNEL_INFO(channel, tail); + + /* Use bounce buffer if the data wraps */ + if (tail + channel->pkt_size >= channel->fifo_size) { + ptr = channel->bounce_buffer; + len = qcom_smd_channel_peek(channel, ptr, channel->pkt_size); + } else { + ptr = channel->rx_fifo + tail; + len = channel->pkt_size; + } + + ret = ept->cb(ept->rpdev, ptr, len, ept->priv, RPMSG_ADDR_ANY); + if (ret < 0) + return ret; + + /* Only forward the tail if the client consumed the data */ + qcom_smd_channel_advance(channel, len); + + channel->pkt_size = 0; + + return 0; +} + +/* + * Per channel interrupt handling + */ +static bool qcom_smd_channel_intr(struct qcom_smd_channel *channel) +{ + bool need_state_scan = false; + int remote_state; + __le32 pktlen; + int avail; + int ret; + + /* Handle state changes */ + remote_state = GET_RX_CHANNEL_INFO(channel, state); + if (remote_state != channel->remote_state) { + channel->remote_state = remote_state; + need_state_scan = true; + + wake_up_interruptible_all(&channel->state_change_event); + } + /* Indicate that we have seen any state change */ + SET_RX_CHANNEL_FLAG(channel, fSTATE, 0); + + /* Signal waiting qcom_smd_send() about the interrupt */ + if (!GET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR)) + wake_up_interruptible_all(&channel->fblockread_event); + + /* Don't consume any data until we've opened the channel */ + if (channel->state != SMD_CHANNEL_OPENED) + goto out; + + /* Indicate that we've seen the new data */ + SET_RX_CHANNEL_FLAG(channel, fHEAD, 0); + + /* Consume data */ + for (;;) { + avail = qcom_smd_channel_get_rx_avail(channel); + + if (!channel->pkt_size && avail >= SMD_PACKET_HEADER_LEN) { + qcom_smd_channel_peek(channel, &pktlen, sizeof(pktlen)); + qcom_smd_channel_advance(channel, SMD_PACKET_HEADER_LEN); + channel->pkt_size = le32_to_cpu(pktlen); + } else if (channel->pkt_size && avail >= channel->pkt_size) { + ret = qcom_smd_channel_recv_single(channel); + if (ret) + break; + } else { + break; + } + } + + /* Indicate that we have seen and updated tail */ + SET_RX_CHANNEL_FLAG(channel, fTAIL, 1); + + /* Signal the remote that we've consumed the data (if requested) */ + if (!GET_RX_CHANNEL_FLAG(channel, fBLOCKREADINTR)) { + /* Ensure ordering of channel info updates */ + wmb(); + + qcom_smd_signal_channel(channel); + } + +out: + return need_state_scan; +} + +/* + * The edge interrupts are triggered by the remote processor on state changes, + * channel info updates or when new channels are created. + */ +static irqreturn_t qcom_smd_edge_intr(int irq, void *data) +{ + struct qcom_smd_edge *edge = data; + struct qcom_smd_channel *channel; + unsigned available; + bool kick_scanner = false; + bool kick_state = false; + + /* + * Handle state changes or data on each of the channels on this edge + */ + spin_lock(&edge->channels_lock); + list_for_each_entry(channel, &edge->channels, list) { + spin_lock(&channel->recv_lock); + kick_state |= qcom_smd_channel_intr(channel); + spin_unlock(&channel->recv_lock); + } + spin_unlock(&edge->channels_lock); + + /* + * Creating a new channel requires allocating an smem entry, so we only + * have to scan if the amount of available space in smem have changed + * since last scan. + */ + available = qcom_smem_get_free_space(edge->remote_pid); + if (available != edge->smem_available) { + edge->smem_available = available; + kick_scanner = true; + } + + if (kick_scanner) + schedule_work(&edge->scan_work); + if (kick_state) + schedule_work(&edge->state_work); + + return IRQ_HANDLED; +} + +/* + * Calculate how much space is available in the tx fifo. + */ +static size_t qcom_smd_get_tx_avail(struct qcom_smd_channel *channel) +{ + unsigned head; + unsigned tail; + unsigned mask = channel->fifo_size - 1; + + head = GET_TX_CHANNEL_INFO(channel, head); + tail = GET_TX_CHANNEL_INFO(channel, tail); + + return mask - ((head - tail) & mask); +} + +/* + * Write count bytes of data into channel, possibly wrapping in the ring buffer + */ +static int qcom_smd_write_fifo(struct qcom_smd_channel *channel, + const void *data, + size_t count) +{ + bool word_aligned; + unsigned head; + size_t len; + + word_aligned = channel->info_word; + head = GET_TX_CHANNEL_INFO(channel, head); + + len = min_t(size_t, count, channel->fifo_size - head); + if (len) { + smd_copy_to_fifo(channel->tx_fifo + head, + data, + len, + word_aligned); + } + + if (len != count) { + smd_copy_to_fifo(channel->tx_fifo, + data + len, + count - len, + word_aligned); + } + + head += count; + head &= (channel->fifo_size - 1); + SET_TX_CHANNEL_INFO(channel, head, head); + + return count; +} + +/** + * __qcom_smd_send - write data to smd channel + * @channel: channel handle + * @data: buffer of data to write + * @len: number of bytes to write + * @wait: flag to indicate if write can wait + * + * This is a blocking write of len bytes into the channel's tx ring buffer and + * signal the remote end. It will sleep until there is enough space available + * in the tx buffer, utilizing the fBLOCKREADINTR signaling mechanism to avoid + * polling. + */ +static int __qcom_smd_send(struct qcom_smd_channel *channel, const void *data, + int len, bool wait) +{ + __le32 hdr[5] = { cpu_to_le32(len), }; + int tlen = sizeof(hdr) + len; + unsigned long flags; + int ret; + + /* Word aligned channels only accept word size aligned data */ + if (channel->info_word && len % 4) + return -EINVAL; + + /* Reject packets that are too big */ + if (tlen >= channel->fifo_size) + return -EINVAL; + + /* Highlight the fact that if we enter the loop below we might sleep */ + if (wait) + might_sleep(); + + spin_lock_irqsave(&channel->tx_lock, flags); + + while (qcom_smd_get_tx_avail(channel) < tlen && + channel->state == SMD_CHANNEL_OPENED) { + if (!wait) { + ret = -EAGAIN; + goto out_unlock; + } + + SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 0); + + /* Wait without holding the tx_lock */ + spin_unlock_irqrestore(&channel->tx_lock, flags); + + ret = wait_event_interruptible(channel->fblockread_event, + qcom_smd_get_tx_avail(channel) >= tlen || + channel->state != SMD_CHANNEL_OPENED); + if (ret) + return ret; + + spin_lock_irqsave(&channel->tx_lock, flags); + + SET_TX_CHANNEL_FLAG(channel, fBLOCKREADINTR, 1); + } + + /* Fail if the channel was closed */ + if (channel->state != SMD_CHANNEL_OPENED) { + ret = -EPIPE; + goto out_unlock; + } + + SET_TX_CHANNEL_FLAG(channel, fTAIL, 0); + + qcom_smd_write_fifo(channel, hdr, sizeof(hdr)); + qcom_smd_write_fifo(channel, data, len); + + SET_TX_CHANNEL_FLAG(channel, fHEAD, 1); + + /* Ensure ordering of channel info updates */ + wmb(); + + qcom_smd_signal_channel(channel); + +out_unlock: + spin_unlock_irqrestore(&channel->tx_lock, flags); + + return ret; +} + +/* + * Helper for opening a channel + */ +static int qcom_smd_channel_open(struct qcom_smd_channel *channel, + rpmsg_rx_cb_t cb) +{ + struct qcom_smd_edge *edge = channel->edge; + size_t bb_size; + int ret; + + /* + * Packets are maximum 4k, but reduce if the fifo is smaller + */ + bb_size = min(channel->fifo_size, SZ_4K); + channel->bounce_buffer = kmalloc(bb_size, GFP_KERNEL); + if (!channel->bounce_buffer) + return -ENOMEM; + + qcom_smd_channel_set_callback(channel, cb); + qcom_smd_channel_set_state(channel, SMD_CHANNEL_OPENING); + + /* Wait for remote to enter opening or opened */ + ret = wait_event_interruptible_timeout(channel->state_change_event, + channel->remote_state == SMD_CHANNEL_OPENING || + channel->remote_state == SMD_CHANNEL_OPENED, + HZ); + if (!ret) { + dev_err(&edge->dev, "remote side did not enter opening state\n"); + goto out_close_timeout; + } + + qcom_smd_channel_set_state(channel, SMD_CHANNEL_OPENED); + + /* Wait for remote to enter opened */ + ret = wait_event_interruptible_timeout(channel->state_change_event, + channel->remote_state == SMD_CHANNEL_OPENED, + HZ); + if (!ret) { + dev_err(&edge->dev, "remote side did not enter open state\n"); + goto out_close_timeout; + } + + return 0; + +out_close_timeout: + qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSED); + return -ETIMEDOUT; +} + +/* + * Helper for closing and resetting a channel + */ +static void qcom_smd_channel_close(struct qcom_smd_channel *channel) +{ + qcom_smd_channel_set_callback(channel, NULL); + + kfree(channel->bounce_buffer); + channel->bounce_buffer = NULL; + + qcom_smd_channel_set_state(channel, SMD_CHANNEL_CLOSED); + qcom_smd_channel_reset(channel); +} + +static struct qcom_smd_channel * +qcom_smd_find_channel(struct qcom_smd_edge *edge, const char *name) +{ + struct qcom_smd_channel *channel; + struct qcom_smd_channel *ret = NULL; + unsigned long flags; + + spin_lock_irqsave(&edge->channels_lock, flags); + list_for_each_entry(channel, &edge->channels, list) { + if (!strcmp(channel->name, name)) { + ret = channel; + break; + } + } + spin_unlock_irqrestore(&edge->channels_lock, flags); + + return ret; +} + +static void __ept_release(struct kref *kref) +{ + struct rpmsg_endpoint *ept = container_of(kref, struct rpmsg_endpoint, + refcount); + kfree(to_smd_endpoint(ept)); +} + +static struct rpmsg_endpoint *qcom_smd_create_ept(struct rpmsg_device *rpdev, + rpmsg_rx_cb_t cb, void *priv, + struct rpmsg_channel_info chinfo) +{ + struct qcom_smd_endpoint *qsept; + struct qcom_smd_channel *channel; + struct qcom_smd_device *qsdev = to_smd_device(rpdev); + struct qcom_smd_edge *edge = qsdev->edge; + struct rpmsg_endpoint *ept; + const char *name = chinfo.name; + int ret; + + /* Wait up to HZ for the channel to appear */ + ret = wait_event_interruptible_timeout(edge->new_channel_event, + (channel = qcom_smd_find_channel(edge, name)) != NULL, + HZ); + if (!ret) + return NULL; + + if (channel->state != SMD_CHANNEL_CLOSED) { + dev_err(&rpdev->dev, "channel %s is busy\n", channel->name); + return NULL; + } + + qsept = kzalloc(sizeof(*qsept), GFP_KERNEL); + if (!qsept) + return NULL; + + ept = &qsept->ept; + + kref_init(&ept->refcount); + + ept->rpdev = rpdev; + ept->cb = cb; + ept->priv = priv; + ept->ops = &qcom_smd_endpoint_ops; + + channel->qsept = qsept; + qsept->qsch = channel; + + ret = qcom_smd_channel_open(channel, cb); + if (ret) + goto free_ept; + + return ept; + +free_ept: + channel->qsept = NULL; + kref_put(&ept->refcount, __ept_release); + return NULL; +} + +static void qcom_smd_destroy_ept(struct rpmsg_endpoint *ept) +{ + struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept); + struct qcom_smd_channel *ch = qsept->qsch; + + qcom_smd_channel_close(ch); + ch->qsept = NULL; + kref_put(&ept->refcount, __ept_release); +} + +static int qcom_smd_send(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept); + + return __qcom_smd_send(qsept->qsch, data, len, true); +} + +static int qcom_smd_trysend(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept); + + return __qcom_smd_send(qsept->qsch, data, len, false); +} + +static int qcom_smd_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) +{ + struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept); + + return __qcom_smd_send(qsept->qsch, data, len, true); +} + +static int qcom_smd_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) +{ + struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept); + + return __qcom_smd_send(qsept->qsch, data, len, false); +} + +static __poll_t qcom_smd_poll(struct rpmsg_endpoint *ept, + struct file *filp, poll_table *wait) +{ + struct qcom_smd_endpoint *qsept = to_smd_endpoint(ept); + struct qcom_smd_channel *channel = qsept->qsch; + __poll_t mask = 0; + + poll_wait(filp, &channel->fblockread_event, wait); + + if (qcom_smd_get_tx_avail(channel) > 20) + mask |= EPOLLOUT | EPOLLWRNORM; + + return mask; +} + +/* + * Finds the device_node for the smd child interested in this channel. + */ +static struct device_node *qcom_smd_match_channel(struct device_node *edge_node, + const char *channel) +{ + struct device_node *child; + const char *name; + const char *key; + int ret; + + for_each_available_child_of_node(edge_node, child) { + key = "qcom,smd-channels"; + ret = of_property_read_string(child, key, &name); + if (ret) + continue; + + if (strcmp(name, channel) == 0) + return child; + } + + return NULL; +} + +static int qcom_smd_announce_create(struct rpmsg_device *rpdev) +{ + struct qcom_smd_endpoint *qept = to_smd_endpoint(rpdev->ept); + struct qcom_smd_channel *channel = qept->qsch; + unsigned long flags; + bool kick_state; + + spin_lock_irqsave(&channel->recv_lock, flags); + kick_state = qcom_smd_channel_intr(channel); + spin_unlock_irqrestore(&channel->recv_lock, flags); + + if (kick_state) + schedule_work(&channel->edge->state_work); + + return 0; +} + +static const struct rpmsg_device_ops qcom_smd_device_ops = { + .create_ept = qcom_smd_create_ept, + .announce_create = qcom_smd_announce_create, +}; + +static const struct rpmsg_endpoint_ops qcom_smd_endpoint_ops = { + .destroy_ept = qcom_smd_destroy_ept, + .send = qcom_smd_send, + .sendto = qcom_smd_sendto, + .trysend = qcom_smd_trysend, + .trysendto = qcom_smd_trysendto, + .poll = qcom_smd_poll, +}; + +static void qcom_smd_release_device(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct qcom_smd_device *qsdev = to_smd_device(rpdev); + + kfree(qsdev); +} + +/* + * Create a smd client device for channel that is being opened. + */ +static int qcom_smd_create_device(struct qcom_smd_channel *channel) +{ + struct qcom_smd_device *qsdev; + struct rpmsg_device *rpdev; + struct qcom_smd_edge *edge = channel->edge; + + dev_dbg(&edge->dev, "registering '%s'\n", channel->name); + + qsdev = kzalloc(sizeof(*qsdev), GFP_KERNEL); + if (!qsdev) + return -ENOMEM; + + /* Link qsdev to our SMD edge */ + qsdev->edge = edge; + + /* Assign callbacks for rpmsg_device */ + qsdev->rpdev.ops = &qcom_smd_device_ops; + + /* Assign public information to the rpmsg_device */ + rpdev = &qsdev->rpdev; + strscpy_pad(rpdev->id.name, channel->name, RPMSG_NAME_SIZE); + rpdev->src = RPMSG_ADDR_ANY; + rpdev->dst = RPMSG_ADDR_ANY; + + rpdev->dev.of_node = qcom_smd_match_channel(edge->of_node, channel->name); + rpdev->dev.parent = &edge->dev; + rpdev->dev.release = qcom_smd_release_device; + + return rpmsg_register_device(rpdev); +} + +static int qcom_smd_create_chrdev(struct qcom_smd_edge *edge) +{ + struct qcom_smd_device *qsdev; + + qsdev = kzalloc(sizeof(*qsdev), GFP_KERNEL); + if (!qsdev) + return -ENOMEM; + + qsdev->edge = edge; + qsdev->rpdev.ops = &qcom_smd_device_ops; + qsdev->rpdev.dev.parent = &edge->dev; + qsdev->rpdev.dev.release = qcom_smd_release_device; + + return rpmsg_ctrldev_register_device(&qsdev->rpdev); +} + +/* + * Allocate the qcom_smd_channel object for a newly found smd channel, + * retrieving and validating the smem items involved. + */ +static struct qcom_smd_channel *qcom_smd_create_channel(struct qcom_smd_edge *edge, + unsigned smem_info_item, + unsigned smem_fifo_item, + char *name) +{ + struct qcom_smd_channel *channel; + size_t fifo_size; + size_t info_size; + void *fifo_base; + void *info; + int ret; + + channel = kzalloc(sizeof(*channel), GFP_KERNEL); + if (!channel) + return ERR_PTR(-ENOMEM); + + channel->edge = edge; + channel->name = kstrdup(name, GFP_KERNEL); + if (!channel->name) { + ret = -ENOMEM; + goto free_channel; + } + + spin_lock_init(&channel->tx_lock); + spin_lock_init(&channel->recv_lock); + init_waitqueue_head(&channel->fblockread_event); + init_waitqueue_head(&channel->state_change_event); + + info = qcom_smem_get(edge->remote_pid, smem_info_item, &info_size); + if (IS_ERR(info)) { + ret = PTR_ERR(info); + goto free_name_and_channel; + } + + /* + * Use the size of the item to figure out which channel info struct to + * use. + */ + if (info_size == 2 * sizeof(struct smd_channel_info_word)) { + channel->info_word = info; + } else if (info_size == 2 * sizeof(struct smd_channel_info)) { + channel->info = info; + } else { + dev_err(&edge->dev, + "channel info of size %zu not supported\n", info_size); + ret = -EINVAL; + goto free_name_and_channel; + } + + fifo_base = qcom_smem_get(edge->remote_pid, smem_fifo_item, &fifo_size); + if (IS_ERR(fifo_base)) { + ret = PTR_ERR(fifo_base); + goto free_name_and_channel; + } + + /* The channel consist of a rx and tx fifo of equal size */ + fifo_size /= 2; + + dev_dbg(&edge->dev, "new channel '%s' info-size: %zu fifo-size: %zu\n", + name, info_size, fifo_size); + + channel->tx_fifo = fifo_base; + channel->rx_fifo = fifo_base + fifo_size; + channel->fifo_size = fifo_size; + + qcom_smd_channel_reset(channel); + + return channel; + +free_name_and_channel: + kfree(channel->name); +free_channel: + kfree(channel); + + return ERR_PTR(ret); +} + +/* + * Scans the allocation table for any newly allocated channels, calls + * qcom_smd_create_channel() to create representations of these and add + * them to the edge's list of channels. + */ +static void qcom_channel_scan_worker(struct work_struct *work) +{ + struct qcom_smd_edge *edge = container_of(work, struct qcom_smd_edge, scan_work); + struct qcom_smd_alloc_entry *alloc_tbl; + struct qcom_smd_alloc_entry *entry; + struct qcom_smd_channel *channel; + unsigned long flags; + unsigned fifo_id; + unsigned info_id; + int tbl; + int i; + u32 eflags, cid; + + for (tbl = 0; tbl < SMD_ALLOC_TBL_COUNT; tbl++) { + alloc_tbl = qcom_smem_get(edge->remote_pid, + smem_items[tbl].alloc_tbl_id, NULL); + if (IS_ERR(alloc_tbl)) + continue; + + for (i = 0; i < SMD_ALLOC_TBL_SIZE; i++) { + entry = &alloc_tbl[i]; + eflags = le32_to_cpu(entry->flags); + if (test_bit(i, edge->allocated[tbl])) + continue; + + if (entry->ref_count == 0) + continue; + + if (!entry->name[0]) + continue; + + if (!(eflags & SMD_CHANNEL_FLAGS_PACKET)) + continue; + + if ((eflags & SMD_CHANNEL_FLAGS_EDGE_MASK) != edge->edge_id) + continue; + + cid = le32_to_cpu(entry->cid); + info_id = smem_items[tbl].info_base_id + cid; + fifo_id = smem_items[tbl].fifo_base_id + cid; + + channel = qcom_smd_create_channel(edge, info_id, fifo_id, entry->name); + if (IS_ERR(channel)) + continue; + + spin_lock_irqsave(&edge->channels_lock, flags); + list_add(&channel->list, &edge->channels); + spin_unlock_irqrestore(&edge->channels_lock, flags); + + dev_dbg(&edge->dev, "new channel found: '%s'\n", channel->name); + set_bit(i, edge->allocated[tbl]); + + wake_up_interruptible_all(&edge->new_channel_event); + } + } + + schedule_work(&edge->state_work); +} + +/* + * This per edge worker scans smem for any new channels and register these. It + * then scans all registered channels for state changes that should be handled + * by creating or destroying smd client devices for the registered channels. + * + * LOCKING: edge->channels_lock only needs to cover the list operations, as the + * worker is killed before any channels are deallocated + */ +static void qcom_channel_state_worker(struct work_struct *work) +{ + struct qcom_smd_channel *channel; + struct qcom_smd_edge *edge = container_of(work, + struct qcom_smd_edge, + state_work); + struct rpmsg_channel_info chinfo; + unsigned remote_state; + unsigned long flags; + + /* + * Register a device for any closed channel where the remote processor + * is showing interest in opening the channel. + */ + spin_lock_irqsave(&edge->channels_lock, flags); + list_for_each_entry(channel, &edge->channels, list) { + if (channel->state != SMD_CHANNEL_CLOSED) + continue; + + /* + * Always open rpm_requests, even when already opened which is + * required on some SoCs like msm8953. + */ + remote_state = GET_RX_CHANNEL_INFO(channel, state); + if (remote_state != SMD_CHANNEL_OPENING && + remote_state != SMD_CHANNEL_OPENED && + strcmp(channel->name, "rpm_requests")) + continue; + + if (channel->registered) + continue; + + spin_unlock_irqrestore(&edge->channels_lock, flags); + qcom_smd_create_device(channel); + spin_lock_irqsave(&edge->channels_lock, flags); + channel->registered = true; + } + + /* + * Unregister the device for any channel that is opened where the + * remote processor is closing the channel. + */ + list_for_each_entry(channel, &edge->channels, list) { + if (channel->state != SMD_CHANNEL_OPENING && + channel->state != SMD_CHANNEL_OPENED) + continue; + + remote_state = GET_RX_CHANNEL_INFO(channel, state); + if (remote_state == SMD_CHANNEL_OPENING || + remote_state == SMD_CHANNEL_OPENED) + continue; + + spin_unlock_irqrestore(&edge->channels_lock, flags); + + strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); + chinfo.src = RPMSG_ADDR_ANY; + chinfo.dst = RPMSG_ADDR_ANY; + rpmsg_unregister_device(&edge->dev, &chinfo); + channel->registered = false; + spin_lock_irqsave(&edge->channels_lock, flags); + } + spin_unlock_irqrestore(&edge->channels_lock, flags); +} + +/* + * Parses an of_node describing an edge. + */ +static int qcom_smd_parse_edge(struct device *dev, + struct device_node *node, + struct qcom_smd_edge *edge) +{ + struct device_node *syscon_np; + const char *key; + int irq; + int ret; + + INIT_LIST_HEAD(&edge->channels); + spin_lock_init(&edge->channels_lock); + + INIT_WORK(&edge->scan_work, qcom_channel_scan_worker); + INIT_WORK(&edge->state_work, qcom_channel_state_worker); + + edge->of_node = of_node_get(node); + + key = "qcom,smd-edge"; + ret = of_property_read_u32(node, key, &edge->edge_id); + if (ret) { + dev_err(dev, "edge missing %s property\n", key); + goto put_node; + } + + edge->remote_pid = QCOM_SMEM_HOST_ANY; + key = "qcom,remote-pid"; + of_property_read_u32(node, key, &edge->remote_pid); + + edge->mbox_client.dev = dev; + edge->mbox_client.knows_txdone = true; + edge->mbox_chan = mbox_request_channel(&edge->mbox_client, 0); + if (IS_ERR(edge->mbox_chan)) { + if (PTR_ERR(edge->mbox_chan) != -ENODEV) { + ret = PTR_ERR(edge->mbox_chan); + goto put_node; + } + + edge->mbox_chan = NULL; + + syscon_np = of_parse_phandle(node, "qcom,ipc", 0); + if (!syscon_np) { + dev_err(dev, "no qcom,ipc node\n"); + ret = -ENODEV; + goto put_node; + } + + edge->ipc_regmap = syscon_node_to_regmap(syscon_np); + of_node_put(syscon_np); + if (IS_ERR(edge->ipc_regmap)) { + ret = PTR_ERR(edge->ipc_regmap); + goto put_node; + } + + key = "qcom,ipc"; + ret = of_property_read_u32_index(node, key, 1, &edge->ipc_offset); + if (ret < 0) { + dev_err(dev, "no offset in %s\n", key); + goto put_node; + } + + ret = of_property_read_u32_index(node, key, 2, &edge->ipc_bit); + if (ret < 0) { + dev_err(dev, "no bit in %s\n", key); + goto put_node; + } + } + + ret = of_property_read_string(node, "label", &edge->name); + if (ret < 0) + edge->name = node->name; + + irq = irq_of_parse_and_map(node, 0); + if (!irq) { + dev_err(dev, "required smd interrupt missing\n"); + ret = -EINVAL; + goto put_node; + } + + ret = devm_request_irq(dev, irq, + qcom_smd_edge_intr, IRQF_TRIGGER_RISING, + node->name, edge); + if (ret) { + dev_err(dev, "failed to request smd irq\n"); + goto put_node; + } + + edge->irq = irq; + + return 0; + +put_node: + of_node_put(node); + edge->of_node = NULL; + + return ret; +} + +/* + * Release function for an edge. + * Reset the state of each associated channel and free the edge context. + */ +static void qcom_smd_edge_release(struct device *dev) +{ + struct qcom_smd_channel *channel, *tmp; + struct qcom_smd_edge *edge = to_smd_edge(dev); + + list_for_each_entry_safe(channel, tmp, &edge->channels, list) { + list_del(&channel->list); + kfree(channel->name); + kfree(channel); + } + + kfree(edge); +} + +static ssize_t rpmsg_name_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct qcom_smd_edge *edge = to_smd_edge(dev); + + return sprintf(buf, "%s\n", edge->name); +} +static DEVICE_ATTR_RO(rpmsg_name); + +static struct attribute *qcom_smd_edge_attrs[] = { + &dev_attr_rpmsg_name.attr, + NULL +}; +ATTRIBUTE_GROUPS(qcom_smd_edge); + +/** + * qcom_smd_register_edge() - register an edge based on an device_node + * @parent: parent device for the edge + * @node: device_node describing the edge + * + * Return: an edge reference, or negative ERR_PTR() on failure. + */ +struct qcom_smd_edge *qcom_smd_register_edge(struct device *parent, + struct device_node *node) +{ + struct qcom_smd_edge *edge; + int ret; + + if (!qcom_smem_is_available()) + return ERR_PTR(-EPROBE_DEFER); + + edge = kzalloc(sizeof(*edge), GFP_KERNEL); + if (!edge) + return ERR_PTR(-ENOMEM); + + init_waitqueue_head(&edge->new_channel_event); + + edge->dev.parent = parent; + edge->dev.release = qcom_smd_edge_release; + edge->dev.of_node = node; + edge->dev.groups = qcom_smd_edge_groups; + dev_set_name(&edge->dev, "%s:%pOFn", dev_name(parent), node); + ret = device_register(&edge->dev); + if (ret) { + pr_err("failed to register smd edge\n"); + put_device(&edge->dev); + return ERR_PTR(ret); + } + + ret = qcom_smd_parse_edge(&edge->dev, node, edge); + if (ret) { + dev_err(&edge->dev, "failed to parse smd edge\n"); + goto unregister_dev; + } + + ret = qcom_smd_create_chrdev(edge); + if (ret) { + dev_err(&edge->dev, "failed to register chrdev for edge\n"); + goto unregister_dev; + } + + schedule_work(&edge->scan_work); + + return edge; + +unregister_dev: + if (!IS_ERR_OR_NULL(edge->mbox_chan)) + mbox_free_channel(edge->mbox_chan); + + device_unregister(&edge->dev); + return ERR_PTR(ret); +} +EXPORT_SYMBOL(qcom_smd_register_edge); + +static int qcom_smd_remove_device(struct device *dev, void *data) +{ + device_unregister(dev); + + return 0; +} + +/** + * qcom_smd_unregister_edge() - release an edge and its children + * @edge: edge reference acquired from qcom_smd_register_edge + */ +void qcom_smd_unregister_edge(struct qcom_smd_edge *edge) +{ + int ret; + + disable_irq(edge->irq); + cancel_work_sync(&edge->scan_work); + cancel_work_sync(&edge->state_work); + + ret = device_for_each_child(&edge->dev, NULL, qcom_smd_remove_device); + if (ret) + dev_warn(&edge->dev, "can't remove smd device: %d\n", ret); + + mbox_free_channel(edge->mbox_chan); + device_unregister(&edge->dev); +} +EXPORT_SYMBOL(qcom_smd_unregister_edge); + +static int qcom_smd_probe(struct platform_device *pdev) +{ + struct device_node *node; + + if (!qcom_smem_is_available()) + return -EPROBE_DEFER; + + for_each_available_child_of_node(pdev->dev.of_node, node) + qcom_smd_register_edge(&pdev->dev, node); + + return 0; +} + +static int qcom_smd_remove_edge(struct device *dev, void *data) +{ + struct qcom_smd_edge *edge = to_smd_edge(dev); + + qcom_smd_unregister_edge(edge); + + return 0; +} + +/* + * Shut down all smd clients by making sure that each edge stops processing + * events and scanning for new channels, then call destroy on the devices. + */ +static void qcom_smd_remove(struct platform_device *pdev) +{ + /* + * qcom_smd_remove_edge always returns zero, so there is no need to + * check the return value of device_for_each_child. + */ + device_for_each_child(&pdev->dev, NULL, qcom_smd_remove_edge); +} + +static const struct of_device_id qcom_smd_of_match[] = { + { .compatible = "qcom,smd" }, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_smd_of_match); + +static struct platform_driver qcom_smd_driver = { + .probe = qcom_smd_probe, + .remove_new = qcom_smd_remove, + .driver = { + .name = "qcom-smd", + .of_match_table = qcom_smd_of_match, + }, +}; + +static int __init qcom_smd_init(void) +{ + return platform_driver_register(&qcom_smd_driver); +} +arch_initcall(qcom_smd_init); + +static void __exit qcom_smd_exit(void) +{ + platform_driver_unregister(&qcom_smd_driver); +} +module_exit(qcom_smd_exit); + +MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>"); +MODULE_DESCRIPTION("Qualcomm Shared Memory Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c new file mode 100644 index 0000000000..09833ad05d --- /dev/null +++ b/drivers/rpmsg/rpmsg_char.c @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022, STMicroelectronics + * Copyright (c) 2016, Linaro Ltd. + * Copyright (c) 2012, Michal Simek <monstr@monstr.eu> + * Copyright (c) 2012, PetaLogix + * Copyright (c) 2011, Texas Instruments, Inc. + * Copyright (c) 2011, Google, Inc. + * + * Based on rpmsg performance statistics driver by Michal Simek, which in turn + * was based on TI & Google OMX rpmsg driver. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/cdev.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/idr.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/poll.h> +#include <linux/rpmsg.h> +#include <linux/skbuff.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <uapi/linux/rpmsg.h> + +#include "rpmsg_char.h" +#include "rpmsg_internal.h" + +#define RPMSG_DEV_MAX (MINORMASK + 1) + +static dev_t rpmsg_major; + +static DEFINE_IDA(rpmsg_ept_ida); +static DEFINE_IDA(rpmsg_minor_ida); + +#define dev_to_eptdev(dev) container_of(dev, struct rpmsg_eptdev, dev) +#define cdev_to_eptdev(i_cdev) container_of(i_cdev, struct rpmsg_eptdev, cdev) + +/** + * struct rpmsg_eptdev - endpoint device context + * @dev: endpoint device + * @cdev: cdev for the endpoint device + * @rpdev: underlaying rpmsg device + * @chinfo: info used to open the endpoint + * @ept_lock: synchronization of @ept modifications + * @ept: rpmsg endpoint reference, when open + * @queue_lock: synchronization of @queue operations + * @queue: incoming message queue + * @readq: wait object for incoming queue + * @default_ept: set to channel default endpoint if the default endpoint should be re-used + * on device open to prevent endpoint address update. + * remote_flow_restricted: to indicate if the remote has requested for flow to be limited + * remote_flow_updated: to indicate if the flow control has been requested + */ +struct rpmsg_eptdev { + struct device dev; + struct cdev cdev; + + struct rpmsg_device *rpdev; + struct rpmsg_channel_info chinfo; + + struct mutex ept_lock; + struct rpmsg_endpoint *ept; + struct rpmsg_endpoint *default_ept; + + spinlock_t queue_lock; + struct sk_buff_head queue; + wait_queue_head_t readq; + + bool remote_flow_restricted; + bool remote_flow_updated; +}; + +int rpmsg_chrdev_eptdev_destroy(struct device *dev, void *data) +{ + struct rpmsg_eptdev *eptdev = dev_to_eptdev(dev); + + mutex_lock(&eptdev->ept_lock); + eptdev->rpdev = NULL; + if (eptdev->ept) { + /* The default endpoint is released by the rpmsg core */ + if (!eptdev->default_ept) + rpmsg_destroy_ept(eptdev->ept); + eptdev->ept = NULL; + } + mutex_unlock(&eptdev->ept_lock); + + /* wake up any blocked readers */ + wake_up_interruptible(&eptdev->readq); + + cdev_device_del(&eptdev->cdev, &eptdev->dev); + put_device(&eptdev->dev); + + return 0; +} +EXPORT_SYMBOL(rpmsg_chrdev_eptdev_destroy); + +static int rpmsg_ept_cb(struct rpmsg_device *rpdev, void *buf, int len, + void *priv, u32 addr) +{ + struct rpmsg_eptdev *eptdev = priv; + struct sk_buff *skb; + + skb = alloc_skb(len, GFP_ATOMIC); + if (!skb) + return -ENOMEM; + + skb_put_data(skb, buf, len); + + spin_lock(&eptdev->queue_lock); + skb_queue_tail(&eptdev->queue, skb); + spin_unlock(&eptdev->queue_lock); + + /* wake up any blocking processes, waiting for new data */ + wake_up_interruptible(&eptdev->readq); + + return 0; +} + +static int rpmsg_ept_flow_cb(struct rpmsg_device *rpdev, void *priv, bool enable) +{ + struct rpmsg_eptdev *eptdev = priv; + + eptdev->remote_flow_restricted = enable; + eptdev->remote_flow_updated = true; + + wake_up_interruptible(&eptdev->readq); + + return 0; +} + +static int rpmsg_eptdev_open(struct inode *inode, struct file *filp) +{ + struct rpmsg_eptdev *eptdev = cdev_to_eptdev(inode->i_cdev); + struct rpmsg_endpoint *ept; + struct rpmsg_device *rpdev = eptdev->rpdev; + struct device *dev = &eptdev->dev; + + mutex_lock(&eptdev->ept_lock); + if (eptdev->ept) { + mutex_unlock(&eptdev->ept_lock); + return -EBUSY; + } + + if (!eptdev->rpdev) { + mutex_unlock(&eptdev->ept_lock); + return -ENETRESET; + } + + get_device(dev); + + /* + * If the default_ept is set, the rpmsg device default endpoint is used. + * Else a new endpoint is created on open that will be destroyed on release. + */ + if (eptdev->default_ept) + ept = eptdev->default_ept; + else + ept = rpmsg_create_ept(rpdev, rpmsg_ept_cb, eptdev, eptdev->chinfo); + + if (!ept) { + dev_err(dev, "failed to open %s\n", eptdev->chinfo.name); + put_device(dev); + mutex_unlock(&eptdev->ept_lock); + return -EINVAL; + } + + ept->flow_cb = rpmsg_ept_flow_cb; + eptdev->ept = ept; + filp->private_data = eptdev; + mutex_unlock(&eptdev->ept_lock); + + return 0; +} + +static int rpmsg_eptdev_release(struct inode *inode, struct file *filp) +{ + struct rpmsg_eptdev *eptdev = cdev_to_eptdev(inode->i_cdev); + struct device *dev = &eptdev->dev; + + /* Close the endpoint, if it's not already destroyed by the parent */ + mutex_lock(&eptdev->ept_lock); + if (eptdev->ept) { + if (!eptdev->default_ept) + rpmsg_destroy_ept(eptdev->ept); + eptdev->ept = NULL; + } + mutex_unlock(&eptdev->ept_lock); + eptdev->remote_flow_updated = false; + + /* Discard all SKBs */ + skb_queue_purge(&eptdev->queue); + + put_device(dev); + + return 0; +} + +static ssize_t rpmsg_eptdev_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *filp = iocb->ki_filp; + struct rpmsg_eptdev *eptdev = filp->private_data; + unsigned long flags; + struct sk_buff *skb; + int use; + + if (!eptdev->ept) + return -EPIPE; + + spin_lock_irqsave(&eptdev->queue_lock, flags); + + /* Wait for data in the queue */ + if (skb_queue_empty(&eptdev->queue)) { + spin_unlock_irqrestore(&eptdev->queue_lock, flags); + + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; + + /* Wait until we get data or the endpoint goes away */ + if (wait_event_interruptible(eptdev->readq, + !skb_queue_empty(&eptdev->queue) || + !eptdev->ept)) + return -ERESTARTSYS; + + /* We lost the endpoint while waiting */ + if (!eptdev->ept) + return -EPIPE; + + spin_lock_irqsave(&eptdev->queue_lock, flags); + } + + skb = skb_dequeue(&eptdev->queue); + spin_unlock_irqrestore(&eptdev->queue_lock, flags); + if (!skb) + return -EFAULT; + + use = min_t(size_t, iov_iter_count(to), skb->len); + if (copy_to_iter(skb->data, use, to) != use) + use = -EFAULT; + + kfree_skb(skb); + + return use; +} + +static ssize_t rpmsg_eptdev_write_iter(struct kiocb *iocb, + struct iov_iter *from) +{ + struct file *filp = iocb->ki_filp; + struct rpmsg_eptdev *eptdev = filp->private_data; + size_t len = iov_iter_count(from); + void *kbuf; + int ret; + + kbuf = kzalloc(len, GFP_KERNEL); + if (!kbuf) + return -ENOMEM; + + if (!copy_from_iter_full(kbuf, len, from)) { + ret = -EFAULT; + goto free_kbuf; + } + + if (mutex_lock_interruptible(&eptdev->ept_lock)) { + ret = -ERESTARTSYS; + goto free_kbuf; + } + + if (!eptdev->ept) { + ret = -EPIPE; + goto unlock_eptdev; + } + + if (filp->f_flags & O_NONBLOCK) { + ret = rpmsg_trysendto(eptdev->ept, kbuf, len, eptdev->chinfo.dst); + if (ret == -ENOMEM) + ret = -EAGAIN; + } else { + ret = rpmsg_sendto(eptdev->ept, kbuf, len, eptdev->chinfo.dst); + } + +unlock_eptdev: + mutex_unlock(&eptdev->ept_lock); + +free_kbuf: + kfree(kbuf); + return ret < 0 ? ret : len; +} + +static __poll_t rpmsg_eptdev_poll(struct file *filp, poll_table *wait) +{ + struct rpmsg_eptdev *eptdev = filp->private_data; + __poll_t mask = 0; + + if (!eptdev->ept) + return EPOLLERR; + + poll_wait(filp, &eptdev->readq, wait); + + if (!skb_queue_empty(&eptdev->queue)) + mask |= EPOLLIN | EPOLLRDNORM; + + if (eptdev->remote_flow_updated) + mask |= EPOLLPRI; + + mutex_lock(&eptdev->ept_lock); + mask |= rpmsg_poll(eptdev->ept, filp, wait); + mutex_unlock(&eptdev->ept_lock); + + return mask; +} + +static long rpmsg_eptdev_ioctl(struct file *fp, unsigned int cmd, + unsigned long arg) +{ + struct rpmsg_eptdev *eptdev = fp->private_data; + + bool set; + int ret; + + switch (cmd) { + case RPMSG_GET_OUTGOING_FLOWCONTROL: + eptdev->remote_flow_updated = false; + ret = put_user(eptdev->remote_flow_restricted, (int __user *)arg); + break; + case RPMSG_SET_INCOMING_FLOWCONTROL: + if (arg > 1) { + ret = -EINVAL; + break; + } + set = !!arg; + ret = rpmsg_set_flow_control(eptdev->ept, set, eptdev->chinfo.dst); + break; + case RPMSG_DESTROY_EPT_IOCTL: + /* Don't allow to destroy a default endpoint. */ + if (eptdev->default_ept) { + ret = -EINVAL; + break; + } + ret = rpmsg_chrdev_eptdev_destroy(&eptdev->dev, NULL); + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static const struct file_operations rpmsg_eptdev_fops = { + .owner = THIS_MODULE, + .open = rpmsg_eptdev_open, + .release = rpmsg_eptdev_release, + .read_iter = rpmsg_eptdev_read_iter, + .write_iter = rpmsg_eptdev_write_iter, + .poll = rpmsg_eptdev_poll, + .unlocked_ioctl = rpmsg_eptdev_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static ssize_t name_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev); + + return sprintf(buf, "%s\n", eptdev->chinfo.name); +} +static DEVICE_ATTR_RO(name); + +static ssize_t src_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", eptdev->chinfo.src); +} +static DEVICE_ATTR_RO(src); + +static ssize_t dst_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", eptdev->chinfo.dst); +} +static DEVICE_ATTR_RO(dst); + +static struct attribute *rpmsg_eptdev_attrs[] = { + &dev_attr_name.attr, + &dev_attr_src.attr, + &dev_attr_dst.attr, + NULL +}; +ATTRIBUTE_GROUPS(rpmsg_eptdev); + +static void rpmsg_eptdev_release_device(struct device *dev) +{ + struct rpmsg_eptdev *eptdev = dev_to_eptdev(dev); + + ida_simple_remove(&rpmsg_ept_ida, dev->id); + ida_simple_remove(&rpmsg_minor_ida, MINOR(eptdev->dev.devt)); + kfree(eptdev); +} + +static struct rpmsg_eptdev *rpmsg_chrdev_eptdev_alloc(struct rpmsg_device *rpdev, + struct device *parent) +{ + struct rpmsg_eptdev *eptdev; + struct device *dev; + + eptdev = kzalloc(sizeof(*eptdev), GFP_KERNEL); + if (!eptdev) + return ERR_PTR(-ENOMEM); + + dev = &eptdev->dev; + eptdev->rpdev = rpdev; + + mutex_init(&eptdev->ept_lock); + spin_lock_init(&eptdev->queue_lock); + skb_queue_head_init(&eptdev->queue); + init_waitqueue_head(&eptdev->readq); + + device_initialize(dev); + dev->class = rpmsg_class; + dev->parent = parent; + dev->groups = rpmsg_eptdev_groups; + dev_set_drvdata(dev, eptdev); + + cdev_init(&eptdev->cdev, &rpmsg_eptdev_fops); + eptdev->cdev.owner = THIS_MODULE; + + return eptdev; +} + +static int rpmsg_chrdev_eptdev_add(struct rpmsg_eptdev *eptdev, struct rpmsg_channel_info chinfo) +{ + struct device *dev = &eptdev->dev; + int ret; + + eptdev->chinfo = chinfo; + + ret = ida_simple_get(&rpmsg_minor_ida, 0, RPMSG_DEV_MAX, GFP_KERNEL); + if (ret < 0) + goto free_eptdev; + dev->devt = MKDEV(MAJOR(rpmsg_major), ret); + + ret = ida_simple_get(&rpmsg_ept_ida, 0, 0, GFP_KERNEL); + if (ret < 0) + goto free_minor_ida; + dev->id = ret; + dev_set_name(dev, "rpmsg%d", ret); + + ret = cdev_device_add(&eptdev->cdev, &eptdev->dev); + if (ret) + goto free_ept_ida; + + /* We can now rely on the release function for cleanup */ + dev->release = rpmsg_eptdev_release_device; + + return ret; + +free_ept_ida: + ida_simple_remove(&rpmsg_ept_ida, dev->id); +free_minor_ida: + ida_simple_remove(&rpmsg_minor_ida, MINOR(dev->devt)); +free_eptdev: + put_device(dev); + kfree(eptdev); + + return ret; +} + +int rpmsg_chrdev_eptdev_create(struct rpmsg_device *rpdev, struct device *parent, + struct rpmsg_channel_info chinfo) +{ + struct rpmsg_eptdev *eptdev; + + eptdev = rpmsg_chrdev_eptdev_alloc(rpdev, parent); + if (IS_ERR(eptdev)) + return PTR_ERR(eptdev); + + return rpmsg_chrdev_eptdev_add(eptdev, chinfo); +} +EXPORT_SYMBOL(rpmsg_chrdev_eptdev_create); + +static int rpmsg_chrdev_probe(struct rpmsg_device *rpdev) +{ + struct rpmsg_channel_info chinfo; + struct rpmsg_eptdev *eptdev; + struct device *dev = &rpdev->dev; + + memcpy(chinfo.name, rpdev->id.name, RPMSG_NAME_SIZE); + chinfo.src = rpdev->src; + chinfo.dst = rpdev->dst; + + eptdev = rpmsg_chrdev_eptdev_alloc(rpdev, dev); + if (IS_ERR(eptdev)) + return PTR_ERR(eptdev); + + /* Set the default_ept to the rpmsg device endpoint */ + eptdev->default_ept = rpdev->ept; + + /* + * The rpmsg_ept_cb uses *priv parameter to get its rpmsg_eptdev context. + * Storedit in default_ept *priv field. + */ + eptdev->default_ept->priv = eptdev; + + return rpmsg_chrdev_eptdev_add(eptdev, chinfo); +} + +static void rpmsg_chrdev_remove(struct rpmsg_device *rpdev) +{ + int ret; + + ret = device_for_each_child(&rpdev->dev, NULL, rpmsg_chrdev_eptdev_destroy); + if (ret) + dev_warn(&rpdev->dev, "failed to destroy endpoints: %d\n", ret); +} + +static struct rpmsg_device_id rpmsg_chrdev_id_table[] = { + { .name = "rpmsg-raw" }, + { }, +}; + +static struct rpmsg_driver rpmsg_chrdev_driver = { + .probe = rpmsg_chrdev_probe, + .remove = rpmsg_chrdev_remove, + .callback = rpmsg_ept_cb, + .id_table = rpmsg_chrdev_id_table, + .drv.name = "rpmsg_chrdev", +}; + +static int rpmsg_chrdev_init(void) +{ + int ret; + + ret = alloc_chrdev_region(&rpmsg_major, 0, RPMSG_DEV_MAX, "rpmsg_char"); + if (ret < 0) { + pr_err("failed to allocate char dev region\n"); + return ret; + } + + ret = register_rpmsg_driver(&rpmsg_chrdev_driver); + if (ret < 0) { + pr_err("rpmsg: failed to register rpmsg raw driver\n"); + goto free_region; + } + + return 0; + +free_region: + unregister_chrdev_region(rpmsg_major, RPMSG_DEV_MAX); + + return ret; +} +postcore_initcall(rpmsg_chrdev_init); + +static void rpmsg_chrdev_exit(void) +{ + unregister_rpmsg_driver(&rpmsg_chrdev_driver); + unregister_chrdev_region(rpmsg_major, RPMSG_DEV_MAX); +} +module_exit(rpmsg_chrdev_exit); + +MODULE_ALIAS("rpmsg:rpmsg_chrdev"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/rpmsg_char.h b/drivers/rpmsg/rpmsg_char.h new file mode 100644 index 0000000000..117d9cbc52 --- /dev/null +++ b/drivers/rpmsg/rpmsg_char.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022, STMicroelectronics + */ + +#ifndef __RPMSG_CHRDEV_H__ +#define __RPMSG_CHRDEV_H__ + +#if IS_ENABLED(CONFIG_RPMSG_CHAR) +/** + * rpmsg_chrdev_eptdev_create() - register char device based on an endpoint + * @rpdev: prepared rpdev to be used for creating endpoints + * @parent: parent device + * @chinfo: associated endpoint channel information. + * + * This function create a new rpmsg char endpoint device to instantiate a new + * endpoint based on chinfo information. + */ +int rpmsg_chrdev_eptdev_create(struct rpmsg_device *rpdev, struct device *parent, + struct rpmsg_channel_info chinfo); + +/** + * rpmsg_chrdev_eptdev_destroy() - destroy created char device endpoint. + * @data: private data associated to the endpoint device + * + * This function destroys a rpmsg char endpoint device created by the RPMSG_DESTROY_EPT_IOCTL + * control. + */ +int rpmsg_chrdev_eptdev_destroy(struct device *dev, void *data); + +#else /*IS_ENABLED(CONFIG_RPMSG_CHAR) */ + +static inline int rpmsg_chrdev_eptdev_create(struct rpmsg_device *rpdev, struct device *parent, + struct rpmsg_channel_info chinfo) +{ + return -ENXIO; +} + +static inline int rpmsg_chrdev_eptdev_destroy(struct device *dev, void *data) +{ + return -ENXIO; +} + +#endif /*IS_ENABLED(CONFIG_RPMSG_CHAR) */ + +#endif /*__RPMSG_CHRDEV_H__ */ diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c new file mode 100644 index 0000000000..32b550c91d --- /dev/null +++ b/drivers/rpmsg/rpmsg_core.c @@ -0,0 +1,741 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * remote processor messaging bus + * + * Copyright (C) 2011 Texas Instruments, Inc. + * Copyright (C) 2011 Google, Inc. + * + * Ohad Ben-Cohen <ohad@wizery.com> + * Brian Swetland <swetland@google.com> + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/rpmsg.h> +#include <linux/of_device.h> +#include <linux/pm_domain.h> +#include <linux/slab.h> + +#include "rpmsg_internal.h" + +struct class *rpmsg_class; +EXPORT_SYMBOL(rpmsg_class); + +/** + * rpmsg_create_channel() - create a new rpmsg channel + * using its name and address info. + * @rpdev: rpmsg device + * @chinfo: channel_info to bind + * + * Return: a pointer to the new rpmsg device on success, or NULL on error. + */ +struct rpmsg_device *rpmsg_create_channel(struct rpmsg_device *rpdev, + struct rpmsg_channel_info *chinfo) +{ + if (WARN_ON(!rpdev)) + return NULL; + if (!rpdev->ops || !rpdev->ops->create_channel) { + dev_err(&rpdev->dev, "no create_channel ops found\n"); + return NULL; + } + + return rpdev->ops->create_channel(rpdev, chinfo); +} +EXPORT_SYMBOL(rpmsg_create_channel); + +/** + * rpmsg_release_channel() - release a rpmsg channel + * using its name and address info. + * @rpdev: rpmsg device + * @chinfo: channel_info to bind + * + * Return: 0 on success or an appropriate error value. + */ +int rpmsg_release_channel(struct rpmsg_device *rpdev, + struct rpmsg_channel_info *chinfo) +{ + if (WARN_ON(!rpdev)) + return -EINVAL; + if (!rpdev->ops || !rpdev->ops->release_channel) { + dev_err(&rpdev->dev, "no release_channel ops found\n"); + return -ENXIO; + } + + return rpdev->ops->release_channel(rpdev, chinfo); +} +EXPORT_SYMBOL(rpmsg_release_channel); + +/** + * rpmsg_create_ept() - create a new rpmsg_endpoint + * @rpdev: rpmsg channel device + * @cb: rx callback handler + * @priv: private data for the driver's use + * @chinfo: channel_info with the local rpmsg address to bind with @cb + * + * Every rpmsg address in the system is bound to an rx callback (so when + * inbound messages arrive, they are dispatched by the rpmsg bus using the + * appropriate callback handler) by means of an rpmsg_endpoint struct. + * + * This function allows drivers to create such an endpoint, and by that, + * bind a callback, and possibly some private data too, to an rpmsg address + * (either one that is known in advance, or one that will be dynamically + * assigned for them). + * + * Simple rpmsg drivers need not call rpmsg_create_ept, because an endpoint + * is already created for them when they are probed by the rpmsg bus + * (using the rx callback provided when they registered to the rpmsg bus). + * + * So things should just work for simple drivers: they already have an + * endpoint, their rx callback is bound to their rpmsg address, and when + * relevant inbound messages arrive (i.e. messages which their dst address + * equals to the src address of their rpmsg channel), the driver's handler + * is invoked to process it. + * + * That said, more complicated drivers might need to allocate + * additional rpmsg addresses, and bind them to different rx callbacks. + * To accomplish that, those drivers need to call this function. + * + * Drivers should provide their @rpdev channel (so the new endpoint would belong + * to the same remote processor their channel belongs to), an rx callback + * function, an optional private data (which is provided back when the + * rx callback is invoked), and an address they want to bind with the + * callback. If @addr is RPMSG_ADDR_ANY, then rpmsg_create_ept will + * dynamically assign them an available rpmsg address (drivers should have + * a very good reason why not to always use RPMSG_ADDR_ANY here). + * + * Return: a pointer to the endpoint on success, or NULL on error. + */ +struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *rpdev, + rpmsg_rx_cb_t cb, void *priv, + struct rpmsg_channel_info chinfo) +{ + if (WARN_ON(!rpdev)) + return NULL; + + return rpdev->ops->create_ept(rpdev, cb, priv, chinfo); +} +EXPORT_SYMBOL(rpmsg_create_ept); + +/** + * rpmsg_destroy_ept() - destroy an existing rpmsg endpoint + * @ept: endpoing to destroy + * + * Should be used by drivers to destroy an rpmsg endpoint previously + * created with rpmsg_create_ept(). As with other types of "free" NULL + * is a valid parameter. + */ +void rpmsg_destroy_ept(struct rpmsg_endpoint *ept) +{ + if (ept && ept->ops) + ept->ops->destroy_ept(ept); +} +EXPORT_SYMBOL(rpmsg_destroy_ept); + +/** + * rpmsg_send() - send a message across to the remote processor + * @ept: the rpmsg endpoint + * @data: payload of message + * @len: length of payload + * + * This function sends @data of length @len on the @ept endpoint. + * The message will be sent to the remote processor which the @ept + * endpoint belongs to, using @ept's address and its associated rpmsg + * device destination addresses. + * In case there are no TX buffers available, the function will block until + * one becomes available, or a timeout of 15 seconds elapses. When the latter + * happens, -ERESTARTSYS is returned. + * + * Can only be called from process context (for now). + * + * Return: 0 on success and an appropriate error value on failure. + */ +int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len) +{ + if (WARN_ON(!ept)) + return -EINVAL; + if (!ept->ops->send) + return -ENXIO; + + return ept->ops->send(ept, data, len); +} +EXPORT_SYMBOL(rpmsg_send); + +/** + * rpmsg_sendto() - send a message across to the remote processor, specify dst + * @ept: the rpmsg endpoint + * @data: payload of message + * @len: length of payload + * @dst: destination address + * + * This function sends @data of length @len to the remote @dst address. + * The message will be sent to the remote processor which the @ept + * endpoint belongs to, using @ept's address as source. + * In case there are no TX buffers available, the function will block until + * one becomes available, or a timeout of 15 seconds elapses. When the latter + * happens, -ERESTARTSYS is returned. + * + * Can only be called from process context (for now). + * + * Return: 0 on success and an appropriate error value on failure. + */ +int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) +{ + if (WARN_ON(!ept)) + return -EINVAL; + if (!ept->ops->sendto) + return -ENXIO; + + return ept->ops->sendto(ept, data, len, dst); +} +EXPORT_SYMBOL(rpmsg_sendto); + +/** + * rpmsg_send_offchannel() - send a message using explicit src/dst addresses + * @ept: the rpmsg endpoint + * @src: source address + * @dst: destination address + * @data: payload of message + * @len: length of payload + * + * This function sends @data of length @len to the remote @dst address, + * and uses @src as the source address. + * The message will be sent to the remote processor which the @ept + * endpoint belongs to. + * In case there are no TX buffers available, the function will block until + * one becomes available, or a timeout of 15 seconds elapses. When the latter + * happens, -ERESTARTSYS is returned. + * + * Can only be called from process context (for now). + * + * Return: 0 on success and an appropriate error value on failure. + */ +int rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst, + void *data, int len) +{ + if (WARN_ON(!ept)) + return -EINVAL; + if (!ept->ops->send_offchannel) + return -ENXIO; + + return ept->ops->send_offchannel(ept, src, dst, data, len); +} +EXPORT_SYMBOL(rpmsg_send_offchannel); + +/** + * rpmsg_trysend() - send a message across to the remote processor + * @ept: the rpmsg endpoint + * @data: payload of message + * @len: length of payload + * + * This function sends @data of length @len on the @ept endpoint. + * The message will be sent to the remote processor which the @ept + * endpoint belongs to, using @ept's address as source and its associated + * rpdev's address as destination. + * In case there are no TX buffers available, the function will immediately + * return -ENOMEM without waiting until one becomes available. + * + * Can only be called from process context (for now). + * + * Return: 0 on success and an appropriate error value on failure. + */ +int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len) +{ + if (WARN_ON(!ept)) + return -EINVAL; + if (!ept->ops->trysend) + return -ENXIO; + + return ept->ops->trysend(ept, data, len); +} +EXPORT_SYMBOL(rpmsg_trysend); + +/** + * rpmsg_trysendto() - send a message across to the remote processor, specify dst + * @ept: the rpmsg endpoint + * @data: payload of message + * @len: length of payload + * @dst: destination address + * + * This function sends @data of length @len to the remote @dst address. + * The message will be sent to the remote processor which the @ept + * endpoint belongs to, using @ept's address as source. + * In case there are no TX buffers available, the function will immediately + * return -ENOMEM without waiting until one becomes available. + * + * Can only be called from process context (for now). + * + * Return: 0 on success and an appropriate error value on failure. + */ +int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) +{ + if (WARN_ON(!ept)) + return -EINVAL; + if (!ept->ops->trysendto) + return -ENXIO; + + return ept->ops->trysendto(ept, data, len, dst); +} +EXPORT_SYMBOL(rpmsg_trysendto); + +/** + * rpmsg_poll() - poll the endpoint's send buffers + * @ept: the rpmsg endpoint + * @filp: file for poll_wait() + * @wait: poll_table for poll_wait() + * + * Return: mask representing the current state of the endpoint's send buffers + */ +__poll_t rpmsg_poll(struct rpmsg_endpoint *ept, struct file *filp, + poll_table *wait) +{ + if (WARN_ON(!ept)) + return 0; + if (!ept->ops->poll) + return 0; + + return ept->ops->poll(ept, filp, wait); +} +EXPORT_SYMBOL(rpmsg_poll); + +/** + * rpmsg_trysend_offchannel() - send a message using explicit src/dst addresses + * @ept: the rpmsg endpoint + * @src: source address + * @dst: destination address + * @data: payload of message + * @len: length of payload + * + * This function sends @data of length @len to the remote @dst address, + * and uses @src as the source address. + * The message will be sent to the remote processor which the @ept + * endpoint belongs to. + * In case there are no TX buffers available, the function will immediately + * return -ENOMEM without waiting until one becomes available. + * + * Can only be called from process context (for now). + * + * Return: 0 on success and an appropriate error value on failure. + */ +int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst, + void *data, int len) +{ + if (WARN_ON(!ept)) + return -EINVAL; + if (!ept->ops->trysend_offchannel) + return -ENXIO; + + return ept->ops->trysend_offchannel(ept, src, dst, data, len); +} +EXPORT_SYMBOL(rpmsg_trysend_offchannel); + +/** + * rpmsg_set_flow_control() - request remote to pause/resume transmission + * @ept: the rpmsg endpoint + * @pause: pause transmission + * @dst: destination address of the endpoint + * + * Return: 0 on success and an appropriate error value on failure. + */ +int rpmsg_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst) +{ + if (WARN_ON(!ept)) + return -EINVAL; + if (!ept->ops->set_flow_control) + return -EOPNOTSUPP; + + return ept->ops->set_flow_control(ept, pause, dst); +} +EXPORT_SYMBOL_GPL(rpmsg_set_flow_control); + +/** + * rpmsg_get_mtu() - get maximum transmission buffer size for sending message. + * @ept: the rpmsg endpoint + * + * This function returns maximum buffer size available for a single outgoing message. + * + * Return: the maximum transmission size on success and an appropriate error + * value on failure. + */ + +ssize_t rpmsg_get_mtu(struct rpmsg_endpoint *ept) +{ + if (WARN_ON(!ept)) + return -EINVAL; + if (!ept->ops->get_mtu) + return -ENOTSUPP; + + return ept->ops->get_mtu(ept); +} +EXPORT_SYMBOL(rpmsg_get_mtu); + +/* + * match a rpmsg channel with a channel info struct. + * this is used to make sure we're not creating rpmsg devices for channels + * that already exist. + */ +static int rpmsg_device_match(struct device *dev, void *data) +{ + struct rpmsg_channel_info *chinfo = data; + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + + if (chinfo->src != RPMSG_ADDR_ANY && chinfo->src != rpdev->src) + return 0; + + if (chinfo->dst != RPMSG_ADDR_ANY && chinfo->dst != rpdev->dst) + return 0; + + if (strncmp(chinfo->name, rpdev->id.name, RPMSG_NAME_SIZE)) + return 0; + + /* found a match ! */ + return 1; +} + +struct device *rpmsg_find_device(struct device *parent, + struct rpmsg_channel_info *chinfo) +{ + return device_find_child(parent, chinfo, rpmsg_device_match); + +} +EXPORT_SYMBOL(rpmsg_find_device); + +/* sysfs show configuration fields */ +#define rpmsg_show_attr(field, path, format_string) \ +static ssize_t \ +field##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ + \ + return sprintf(buf, format_string, rpdev->path); \ +} \ +static DEVICE_ATTR_RO(field); + +#define rpmsg_string_attr(field, member) \ +static ssize_t \ +field##_store(struct device *dev, struct device_attribute *attr, \ + const char *buf, size_t sz) \ +{ \ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ + const char *old; \ + char *new; \ + \ + new = kstrndup(buf, sz, GFP_KERNEL); \ + if (!new) \ + return -ENOMEM; \ + new[strcspn(new, "\n")] = '\0'; \ + \ + device_lock(dev); \ + old = rpdev->member; \ + if (strlen(new)) { \ + rpdev->member = new; \ + } else { \ + kfree(new); \ + rpdev->member = NULL; \ + } \ + device_unlock(dev); \ + \ + kfree(old); \ + \ + return sz; \ +} \ +static ssize_t \ +field##_show(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); \ + \ + return sprintf(buf, "%s\n", rpdev->member); \ +} \ +static DEVICE_ATTR_RW(field) + +/* for more info, see Documentation/ABI/testing/sysfs-bus-rpmsg */ +rpmsg_show_attr(name, id.name, "%s\n"); +rpmsg_show_attr(src, src, "0x%x\n"); +rpmsg_show_attr(dst, dst, "0x%x\n"); +rpmsg_show_attr(announce, announce ? "true" : "false", "%s\n"); +rpmsg_string_attr(driver_override, driver_override); + +static ssize_t modalias_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + ssize_t len; + + len = of_device_modalias(dev, buf, PAGE_SIZE); + if (len != -ENODEV) + return len; + + return sprintf(buf, RPMSG_DEVICE_MODALIAS_FMT "\n", rpdev->id.name); +} +static DEVICE_ATTR_RO(modalias); + +static struct attribute *rpmsg_dev_attrs[] = { + &dev_attr_name.attr, + &dev_attr_modalias.attr, + &dev_attr_dst.attr, + &dev_attr_src.attr, + &dev_attr_announce.attr, + &dev_attr_driver_override.attr, + NULL, +}; +ATTRIBUTE_GROUPS(rpmsg_dev); + +/* rpmsg devices and drivers are matched using the service name */ +static inline int rpmsg_id_match(const struct rpmsg_device *rpdev, + const struct rpmsg_device_id *id) +{ + return strncmp(id->name, rpdev->id.name, RPMSG_NAME_SIZE) == 0; +} + +/* match rpmsg channel and rpmsg driver */ +static int rpmsg_dev_match(struct device *dev, struct device_driver *drv) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct rpmsg_driver *rpdrv = to_rpmsg_driver(drv); + const struct rpmsg_device_id *ids = rpdrv->id_table; + unsigned int i; + + if (rpdev->driver_override) + return !strcmp(rpdev->driver_override, drv->name); + + if (ids) + for (i = 0; ids[i].name[0]; i++) + if (rpmsg_id_match(rpdev, &ids[i])) { + rpdev->id.driver_data = ids[i].driver_data; + return 1; + } + + return of_driver_match_device(dev, drv); +} + +static int rpmsg_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + const struct rpmsg_device *rpdev = to_rpmsg_device(dev); + int ret; + + ret = of_device_uevent_modalias(dev, env); + if (ret != -ENODEV) + return ret; + + return add_uevent_var(env, "MODALIAS=" RPMSG_DEVICE_MODALIAS_FMT, + rpdev->id.name); +} + +/* + * when an rpmsg driver is probed with a channel, we seamlessly create + * it an endpoint, binding its rx callback to a unique local rpmsg + * address. + * + * if we need to, we also announce about this channel to the remote + * processor (needed in case the driver is exposing an rpmsg service). + */ +static int rpmsg_dev_probe(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver); + struct rpmsg_channel_info chinfo = {}; + struct rpmsg_endpoint *ept = NULL; + int err; + + err = dev_pm_domain_attach(dev, true); + if (err) + goto out; + + if (rpdrv->callback) { + strncpy(chinfo.name, rpdev->id.name, RPMSG_NAME_SIZE); + chinfo.src = rpdev->src; + chinfo.dst = RPMSG_ADDR_ANY; + + ept = rpmsg_create_ept(rpdev, rpdrv->callback, NULL, chinfo); + if (!ept) { + dev_err(dev, "failed to create endpoint\n"); + err = -ENOMEM; + goto out; + } + + rpdev->ept = ept; + rpdev->src = ept->addr; + + ept->flow_cb = rpdrv->flowcontrol; + } + + err = rpdrv->probe(rpdev); + if (err) { + dev_err(dev, "%s: failed: %d\n", __func__, err); + goto destroy_ept; + } + + if (ept && rpdev->ops->announce_create) { + err = rpdev->ops->announce_create(rpdev); + if (err) { + dev_err(dev, "failed to announce creation\n"); + goto remove_rpdev; + } + } + + return 0; + +remove_rpdev: + if (rpdrv->remove) + rpdrv->remove(rpdev); +destroy_ept: + if (ept) + rpmsg_destroy_ept(ept); +out: + return err; +} + +static void rpmsg_dev_remove(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver); + + if (rpdev->ops->announce_destroy) + rpdev->ops->announce_destroy(rpdev); + + if (rpdrv->remove) + rpdrv->remove(rpdev); + + dev_pm_domain_detach(dev, true); + + if (rpdev->ept) + rpmsg_destroy_ept(rpdev->ept); +} + +static struct bus_type rpmsg_bus = { + .name = "rpmsg", + .match = rpmsg_dev_match, + .dev_groups = rpmsg_dev_groups, + .uevent = rpmsg_uevent, + .probe = rpmsg_dev_probe, + .remove = rpmsg_dev_remove, +}; + +/* + * A helper for registering rpmsg device with driver override and name. + * Drivers should not be using it, but instead rpmsg_register_device(). + */ +int rpmsg_register_device_override(struct rpmsg_device *rpdev, + const char *driver_override) +{ + struct device *dev = &rpdev->dev; + int ret; + + if (driver_override) + strscpy_pad(rpdev->id.name, driver_override, RPMSG_NAME_SIZE); + + dev_set_name(dev, "%s.%s.%d.%d", dev_name(dev->parent), + rpdev->id.name, rpdev->src, rpdev->dst); + + dev->bus = &rpmsg_bus; + + device_initialize(dev); + if (driver_override) { + ret = driver_set_override(dev, &rpdev->driver_override, + driver_override, + strlen(driver_override)); + if (ret) { + dev_err(dev, "device_set_override failed: %d\n", ret); + put_device(dev); + return ret; + } + } + + ret = device_add(dev); + if (ret) { + dev_err(dev, "device_add failed: %d\n", ret); + kfree(rpdev->driver_override); + rpdev->driver_override = NULL; + put_device(dev); + } + + return ret; +} +EXPORT_SYMBOL(rpmsg_register_device_override); + +int rpmsg_register_device(struct rpmsg_device *rpdev) +{ + return rpmsg_register_device_override(rpdev, NULL); +} +EXPORT_SYMBOL(rpmsg_register_device); + +/* + * find an existing channel using its name + address properties, + * and destroy it + */ +int rpmsg_unregister_device(struct device *parent, + struct rpmsg_channel_info *chinfo) +{ + struct device *dev; + + dev = rpmsg_find_device(parent, chinfo); + if (!dev) + return -EINVAL; + + device_unregister(dev); + + put_device(dev); + + return 0; +} +EXPORT_SYMBOL(rpmsg_unregister_device); + +/** + * __register_rpmsg_driver() - register an rpmsg driver with the rpmsg bus + * @rpdrv: pointer to a struct rpmsg_driver + * @owner: owning module/driver + * + * Return: 0 on success, and an appropriate error value on failure. + */ +int __register_rpmsg_driver(struct rpmsg_driver *rpdrv, struct module *owner) +{ + rpdrv->drv.bus = &rpmsg_bus; + rpdrv->drv.owner = owner; + return driver_register(&rpdrv->drv); +} +EXPORT_SYMBOL(__register_rpmsg_driver); + +/** + * unregister_rpmsg_driver() - unregister an rpmsg driver from the rpmsg bus + * @rpdrv: pointer to a struct rpmsg_driver + * + * Return: 0 on success, and an appropriate error value on failure. + */ +void unregister_rpmsg_driver(struct rpmsg_driver *rpdrv) +{ + driver_unregister(&rpdrv->drv); +} +EXPORT_SYMBOL(unregister_rpmsg_driver); + + +static int __init rpmsg_init(void) +{ + int ret; + + rpmsg_class = class_create("rpmsg"); + if (IS_ERR(rpmsg_class)) { + pr_err("failed to create rpmsg class\n"); + return PTR_ERR(rpmsg_class); + } + + ret = bus_register(&rpmsg_bus); + if (ret) { + pr_err("failed to register rpmsg bus: %d\n", ret); + class_destroy(rpmsg_class); + } + return ret; +} +postcore_initcall(rpmsg_init); + +static void __exit rpmsg_fini(void) +{ + bus_unregister(&rpmsg_bus); + class_destroy(rpmsg_class); +} +module_exit(rpmsg_fini); + +MODULE_DESCRIPTION("remote processor messaging bus"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/rpmsg_ctrl.c b/drivers/rpmsg/rpmsg_ctrl.c new file mode 100644 index 0000000000..4332538356 --- /dev/null +++ b/drivers/rpmsg/rpmsg_ctrl.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022, STMicroelectronics + * Copyright (c) 2016, Linaro Ltd. + * Copyright (c) 2012, Michal Simek <monstr@monstr.eu> + * Copyright (c) 2012, PetaLogix + * Copyright (c) 2011, Texas Instruments, Inc. + * Copyright (c) 2011, Google, Inc. + * + * Based on rpmsg performance statistics driver by Michal Simek, which in turn + * was based on TI & Google OMX rpmsg driver. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/cdev.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/idr.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/rpmsg.h> +#include <linux/skbuff.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <uapi/linux/rpmsg.h> + +#include "rpmsg_char.h" +#include "rpmsg_internal.h" + +#define RPMSG_DEV_MAX (MINORMASK + 1) + +static dev_t rpmsg_major; + +static DEFINE_IDA(rpmsg_ctrl_ida); +static DEFINE_IDA(rpmsg_minor_ida); + +#define dev_to_ctrldev(dev) container_of(dev, struct rpmsg_ctrldev, dev) +#define cdev_to_ctrldev(i_cdev) container_of(i_cdev, struct rpmsg_ctrldev, cdev) + +/** + * struct rpmsg_ctrldev - control device for instantiating endpoint devices + * @rpdev: underlaying rpmsg device + * @cdev: cdev for the ctrl device + * @dev: device for the ctrl device + * @ctrl_lock: serialize the ioctrls. + */ +struct rpmsg_ctrldev { + struct rpmsg_device *rpdev; + struct cdev cdev; + struct device dev; + struct mutex ctrl_lock; +}; + +static int rpmsg_ctrldev_open(struct inode *inode, struct file *filp) +{ + struct rpmsg_ctrldev *ctrldev = cdev_to_ctrldev(inode->i_cdev); + + get_device(&ctrldev->dev); + filp->private_data = ctrldev; + + return 0; +} + +static int rpmsg_ctrldev_release(struct inode *inode, struct file *filp) +{ + struct rpmsg_ctrldev *ctrldev = cdev_to_ctrldev(inode->i_cdev); + + put_device(&ctrldev->dev); + + return 0; +} + +static long rpmsg_ctrldev_ioctl(struct file *fp, unsigned int cmd, + unsigned long arg) +{ + struct rpmsg_ctrldev *ctrldev = fp->private_data; + void __user *argp = (void __user *)arg; + struct rpmsg_endpoint_info eptinfo; + struct rpmsg_channel_info chinfo; + struct rpmsg_device *rpdev; + int ret = 0; + + if (copy_from_user(&eptinfo, argp, sizeof(eptinfo))) + return -EFAULT; + + memcpy(chinfo.name, eptinfo.name, RPMSG_NAME_SIZE); + chinfo.name[RPMSG_NAME_SIZE - 1] = '\0'; + chinfo.src = eptinfo.src; + chinfo.dst = eptinfo.dst; + + mutex_lock(&ctrldev->ctrl_lock); + switch (cmd) { + case RPMSG_CREATE_EPT_IOCTL: + ret = rpmsg_chrdev_eptdev_create(ctrldev->rpdev, &ctrldev->dev, chinfo); + break; + + case RPMSG_CREATE_DEV_IOCTL: + rpdev = rpmsg_create_channel(ctrldev->rpdev, &chinfo); + if (!rpdev) { + dev_err(&ctrldev->dev, "failed to create %s channel\n", chinfo.name); + ret = -ENXIO; + } + break; + + case RPMSG_RELEASE_DEV_IOCTL: + ret = rpmsg_release_channel(ctrldev->rpdev, &chinfo); + if (ret) + dev_err(&ctrldev->dev, "failed to release %s channel (%d)\n", + chinfo.name, ret); + break; + + default: + ret = -EINVAL; + } + mutex_unlock(&ctrldev->ctrl_lock); + + return ret; +}; + +static const struct file_operations rpmsg_ctrldev_fops = { + .owner = THIS_MODULE, + .open = rpmsg_ctrldev_open, + .release = rpmsg_ctrldev_release, + .unlocked_ioctl = rpmsg_ctrldev_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static void rpmsg_ctrldev_release_device(struct device *dev) +{ + struct rpmsg_ctrldev *ctrldev = dev_to_ctrldev(dev); + + ida_simple_remove(&rpmsg_ctrl_ida, dev->id); + ida_simple_remove(&rpmsg_minor_ida, MINOR(dev->devt)); + kfree(ctrldev); +} + +static int rpmsg_ctrldev_probe(struct rpmsg_device *rpdev) +{ + struct rpmsg_ctrldev *ctrldev; + struct device *dev; + int ret; + + ctrldev = kzalloc(sizeof(*ctrldev), GFP_KERNEL); + if (!ctrldev) + return -ENOMEM; + + ctrldev->rpdev = rpdev; + + dev = &ctrldev->dev; + device_initialize(dev); + dev->parent = &rpdev->dev; + dev->class = rpmsg_class; + + mutex_init(&ctrldev->ctrl_lock); + cdev_init(&ctrldev->cdev, &rpmsg_ctrldev_fops); + ctrldev->cdev.owner = THIS_MODULE; + + ret = ida_simple_get(&rpmsg_minor_ida, 0, RPMSG_DEV_MAX, GFP_KERNEL); + if (ret < 0) + goto free_ctrldev; + dev->devt = MKDEV(MAJOR(rpmsg_major), ret); + + ret = ida_simple_get(&rpmsg_ctrl_ida, 0, 0, GFP_KERNEL); + if (ret < 0) + goto free_minor_ida; + dev->id = ret; + dev_set_name(&ctrldev->dev, "rpmsg_ctrl%d", ret); + + ret = cdev_device_add(&ctrldev->cdev, &ctrldev->dev); + if (ret) + goto free_ctrl_ida; + + /* We can now rely on the release function for cleanup */ + dev->release = rpmsg_ctrldev_release_device; + + dev_set_drvdata(&rpdev->dev, ctrldev); + + return ret; + +free_ctrl_ida: + ida_simple_remove(&rpmsg_ctrl_ida, dev->id); +free_minor_ida: + ida_simple_remove(&rpmsg_minor_ida, MINOR(dev->devt)); +free_ctrldev: + put_device(dev); + kfree(ctrldev); + + return ret; +} + +static void rpmsg_ctrldev_remove(struct rpmsg_device *rpdev) +{ + struct rpmsg_ctrldev *ctrldev = dev_get_drvdata(&rpdev->dev); + int ret; + + mutex_lock(&ctrldev->ctrl_lock); + /* Destroy all endpoints */ + ret = device_for_each_child(&ctrldev->dev, NULL, rpmsg_chrdev_eptdev_destroy); + if (ret) + dev_warn(&rpdev->dev, "failed to nuke endpoints: %d\n", ret); + mutex_unlock(&ctrldev->ctrl_lock); + + cdev_device_del(&ctrldev->cdev, &ctrldev->dev); + put_device(&ctrldev->dev); +} + +static struct rpmsg_driver rpmsg_ctrldev_driver = { + .probe = rpmsg_ctrldev_probe, + .remove = rpmsg_ctrldev_remove, + .drv = { + .name = "rpmsg_ctrl", + }, +}; + +static int rpmsg_ctrldev_init(void) +{ + int ret; + + ret = alloc_chrdev_region(&rpmsg_major, 0, RPMSG_DEV_MAX, "rpmsg_ctrl"); + if (ret < 0) { + pr_err("failed to allocate char dev region\n"); + return ret; + } + + ret = register_rpmsg_driver(&rpmsg_ctrldev_driver); + if (ret < 0) { + pr_err("failed to register rpmsg driver\n"); + unregister_chrdev_region(rpmsg_major, RPMSG_DEV_MAX); + } + + return ret; +} +postcore_initcall(rpmsg_ctrldev_init); + +static void rpmsg_ctrldev_exit(void) +{ + unregister_rpmsg_driver(&rpmsg_ctrldev_driver); + unregister_chrdev_region(rpmsg_major, RPMSG_DEV_MAX); +} +module_exit(rpmsg_ctrldev_exit); + +MODULE_DESCRIPTION("rpmsg control interface"); +MODULE_ALIAS("rpmsg:" KBUILD_MODNAME); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/rpmsg_internal.h b/drivers/rpmsg/rpmsg_internal.h new file mode 100644 index 0000000000..b950d6f790 --- /dev/null +++ b/drivers/rpmsg/rpmsg_internal.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * remote processor messaging bus internals + * + * Copyright (C) 2011 Texas Instruments, Inc. + * Copyright (C) 2011 Google, Inc. + * + * Ohad Ben-Cohen <ohad@wizery.com> + * Brian Swetland <swetland@google.com> + */ + +#ifndef __RPMSG_INTERNAL_H__ +#define __RPMSG_INTERNAL_H__ + +#include <linux/rpmsg.h> +#include <linux/poll.h> + +#define to_rpmsg_device(d) container_of(d, struct rpmsg_device, dev) +#define to_rpmsg_driver(d) container_of(d, struct rpmsg_driver, drv) + +extern struct class *rpmsg_class; + +/** + * struct rpmsg_device_ops - indirection table for the rpmsg_device operations + * @create_channel: create backend-specific channel, optional + * @release_channel: release backend-specific channel, optional + * @create_ept: create backend-specific endpoint, required + * @announce_create: announce presence of new channel, optional + * @announce_destroy: announce destruction of channel, optional + * + * Indirection table for the operations that a rpmsg backend should implement. + * @announce_create and @announce_destroy are optional as the backend might + * advertise new channels implicitly by creating the endpoints. + */ +struct rpmsg_device_ops { + struct rpmsg_device *(*create_channel)(struct rpmsg_device *rpdev, + struct rpmsg_channel_info *chinfo); + int (*release_channel)(struct rpmsg_device *rpdev, + struct rpmsg_channel_info *chinfo); + struct rpmsg_endpoint *(*create_ept)(struct rpmsg_device *rpdev, + rpmsg_rx_cb_t cb, void *priv, + struct rpmsg_channel_info chinfo); + + int (*announce_create)(struct rpmsg_device *rpdev); + int (*announce_destroy)(struct rpmsg_device *rpdev); +}; + +/** + * struct rpmsg_endpoint_ops - indirection table for rpmsg_endpoint operations + * @destroy_ept: see @rpmsg_destroy_ept(), required + * @send: see @rpmsg_send(), required + * @sendto: see @rpmsg_sendto(), optional + * @send_offchannel: see @rpmsg_send_offchannel(), optional + * @trysend: see @rpmsg_trysend(), required + * @trysendto: see @rpmsg_trysendto(), optional + * @trysend_offchannel: see @rpmsg_trysend_offchannel(), optional + * @poll: see @rpmsg_poll(), optional + * @set_flow_control: see @rpmsg_set_flow_control(), optional + * @get_mtu: see @rpmsg_get_mtu(), optional + * + * Indirection table for the operations that a rpmsg backend should implement. + * In addition to @destroy_ept, the backend must at least implement @send and + * @trysend, while the variants sending data off-channel are optional. + */ +struct rpmsg_endpoint_ops { + void (*destroy_ept)(struct rpmsg_endpoint *ept); + + int (*send)(struct rpmsg_endpoint *ept, void *data, int len); + int (*sendto)(struct rpmsg_endpoint *ept, void *data, int len, u32 dst); + int (*send_offchannel)(struct rpmsg_endpoint *ept, u32 src, u32 dst, + void *data, int len); + + int (*trysend)(struct rpmsg_endpoint *ept, void *data, int len); + int (*trysendto)(struct rpmsg_endpoint *ept, void *data, int len, u32 dst); + int (*trysend_offchannel)(struct rpmsg_endpoint *ept, u32 src, u32 dst, + void *data, int len); + __poll_t (*poll)(struct rpmsg_endpoint *ept, struct file *filp, + poll_table *wait); + int (*set_flow_control)(struct rpmsg_endpoint *ept, bool pause, u32 dst); + ssize_t (*get_mtu)(struct rpmsg_endpoint *ept); +}; + +struct device *rpmsg_find_device(struct device *parent, + struct rpmsg_channel_info *chinfo); + +struct rpmsg_device *rpmsg_create_channel(struct rpmsg_device *rpdev, + struct rpmsg_channel_info *chinfo); +int rpmsg_release_channel(struct rpmsg_device *rpdev, + struct rpmsg_channel_info *chinfo); +/** + * rpmsg_ctrldev_register_device() - register a char device for control based on rpdev + * @rpdev: prepared rpdev to be used for creating endpoints + * + * This function wraps rpmsg_register_device() preparing the rpdev for use as + * basis for the rpmsg chrdev. + */ +static inline int rpmsg_ctrldev_register_device(struct rpmsg_device *rpdev) +{ + return rpmsg_register_device_override(rpdev, "rpmsg_ctrl"); +} + +#endif diff --git a/drivers/rpmsg/rpmsg_ns.c b/drivers/rpmsg/rpmsg_ns.c new file mode 100644 index 0000000000..c70ad03ff2 --- /dev/null +++ b/drivers/rpmsg/rpmsg_ns.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) STMicroelectronics 2020 - All Rights Reserved + */ +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/rpmsg.h> +#include <linux/rpmsg/ns.h> +#include <linux/slab.h> + +#include "rpmsg_internal.h" + +/** + * rpmsg_ns_register_device() - register name service device based on rpdev + * @rpdev: prepared rpdev to be used for creating endpoints + * + * This function wraps rpmsg_register_device() preparing the rpdev for use as + * basis for the rpmsg name service device. + */ +int rpmsg_ns_register_device(struct rpmsg_device *rpdev) +{ + rpdev->src = RPMSG_NS_ADDR; + rpdev->dst = RPMSG_NS_ADDR; + + return rpmsg_register_device_override(rpdev, "rpmsg_ns"); +} +EXPORT_SYMBOL(rpmsg_ns_register_device); + +/* invoked when a name service announcement arrives */ +static int rpmsg_ns_cb(struct rpmsg_device *rpdev, void *data, int len, + void *priv, u32 src) +{ + struct rpmsg_ns_msg *msg = data; + struct rpmsg_device *newch; + struct rpmsg_channel_info chinfo; + struct device *dev = rpdev->dev.parent; + int ret; + +#if defined(CONFIG_DYNAMIC_DEBUG) + dynamic_hex_dump("NS announcement: ", DUMP_PREFIX_NONE, 16, 1, + data, len, true); +#endif + + if (len != sizeof(*msg)) { + dev_err(dev, "malformed ns msg (%d)\n", len); + return -EINVAL; + } + + /* don't trust the remote processor for null terminating the name */ + msg->name[RPMSG_NAME_SIZE - 1] = '\0'; + + strncpy(chinfo.name, msg->name, sizeof(chinfo.name)); + chinfo.src = RPMSG_ADDR_ANY; + chinfo.dst = rpmsg32_to_cpu(rpdev, msg->addr); + + dev_info(dev, "%sing channel %s addr 0x%x\n", + rpmsg32_to_cpu(rpdev, msg->flags) & RPMSG_NS_DESTROY ? + "destroy" : "creat", msg->name, chinfo.dst); + + if (rpmsg32_to_cpu(rpdev, msg->flags) & RPMSG_NS_DESTROY) { + ret = rpmsg_release_channel(rpdev, &chinfo); + if (ret) + dev_err(dev, "rpmsg_destroy_channel failed: %d\n", ret); + } else { + newch = rpmsg_create_channel(rpdev, &chinfo); + if (!newch) + dev_err(dev, "rpmsg_create_channel failed\n"); + } + + return 0; +} + +static int rpmsg_ns_probe(struct rpmsg_device *rpdev) +{ + struct rpmsg_endpoint *ns_ept; + struct rpmsg_channel_info ns_chinfo = { + .src = RPMSG_NS_ADDR, + .dst = RPMSG_NS_ADDR, + .name = "name_service", + }; + + /* + * Create the NS announcement service endpoint associated to the RPMsg + * device. The endpoint will be automatically destroyed when the RPMsg + * device will be deleted. + */ + ns_ept = rpmsg_create_ept(rpdev, rpmsg_ns_cb, NULL, ns_chinfo); + if (!ns_ept) { + dev_err(&rpdev->dev, "failed to create the ns ept\n"); + return -ENOMEM; + } + rpdev->ept = ns_ept; + + return 0; +} + +static struct rpmsg_driver rpmsg_ns_driver = { + .drv.name = KBUILD_MODNAME, + .probe = rpmsg_ns_probe, +}; + +static int rpmsg_ns_init(void) +{ + int ret; + + ret = register_rpmsg_driver(&rpmsg_ns_driver); + if (ret < 0) + pr_err("%s: Failed to register rpmsg driver\n", __func__); + + return ret; +} +postcore_initcall(rpmsg_ns_init); + +static void rpmsg_ns_exit(void) +{ + unregister_rpmsg_driver(&rpmsg_ns_driver); +} +module_exit(rpmsg_ns_exit); + +MODULE_DESCRIPTION("Name service announcement rpmsg driver"); +MODULE_AUTHOR("Arnaud Pouliquen <arnaud.pouliquen@st.com>"); +MODULE_ALIAS("rpmsg:" KBUILD_MODNAME); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c new file mode 100644 index 0000000000..f1af0f6746 --- /dev/null +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -0,0 +1,1082 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Virtio-based remote processor messaging bus + * + * Copyright (C) 2011 Texas Instruments, Inc. + * Copyright (C) 2011 Google, Inc. + * + * Ohad Ben-Cohen <ohad@wizery.com> + * Brian Swetland <swetland@google.com> + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include <linux/dma-mapping.h> +#include <linux/idr.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/rpmsg.h> +#include <linux/rpmsg/byteorder.h> +#include <linux/rpmsg/ns.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/sched.h> +#include <linux/virtio.h> +#include <linux/virtio_ids.h> +#include <linux/virtio_config.h> +#include <linux/wait.h> + +#include "rpmsg_internal.h" + +/** + * struct virtproc_info - virtual remote processor state + * @vdev: the virtio device + * @rvq: rx virtqueue + * @svq: tx virtqueue + * @rbufs: kernel address of rx buffers + * @sbufs: kernel address of tx buffers + * @num_bufs: total number of buffers for rx and tx + * @buf_size: size of one rx or tx buffer + * @last_sbuf: index of last tx buffer used + * @bufs_dma: dma base addr of the buffers + * @tx_lock: protects svq, sbufs and sleepers, to allow concurrent senders. + * sending a message might require waking up a dozing remote + * processor, which involves sleeping, hence the mutex. + * @endpoints: idr of local endpoints, allows fast retrieval + * @endpoints_lock: lock of the endpoints set + * @sendq: wait queue of sending contexts waiting for a tx buffers + * @sleepers: number of senders that are waiting for a tx buffer + * + * This structure stores the rpmsg state of a given virtio remote processor + * device (there might be several virtio proc devices for each physical + * remote processor). + */ +struct virtproc_info { + struct virtio_device *vdev; + struct virtqueue *rvq, *svq; + void *rbufs, *sbufs; + unsigned int num_bufs; + unsigned int buf_size; + int last_sbuf; + dma_addr_t bufs_dma; + struct mutex tx_lock; + struct idr endpoints; + struct mutex endpoints_lock; + wait_queue_head_t sendq; + atomic_t sleepers; +}; + +/* The feature bitmap for virtio rpmsg */ +#define VIRTIO_RPMSG_F_NS 0 /* RP supports name service notifications */ + +/** + * struct rpmsg_hdr - common header for all rpmsg messages + * @src: source address + * @dst: destination address + * @reserved: reserved for future use + * @len: length of payload (in bytes) + * @flags: message flags + * @data: @len bytes of message payload data + * + * Every message sent(/received) on the rpmsg bus begins with this header. + */ +struct rpmsg_hdr { + __rpmsg32 src; + __rpmsg32 dst; + __rpmsg32 reserved; + __rpmsg16 len; + __rpmsg16 flags; + u8 data[]; +} __packed; + + +/** + * struct virtio_rpmsg_channel - rpmsg channel descriptor + * @rpdev: the rpmsg channel device + * @vrp: the virtio remote processor device this channel belongs to + * + * This structure stores the channel that links the rpmsg device to the virtio + * remote processor device. + */ +struct virtio_rpmsg_channel { + struct rpmsg_device rpdev; + + struct virtproc_info *vrp; +}; + +#define to_virtio_rpmsg_channel(_rpdev) \ + container_of(_rpdev, struct virtio_rpmsg_channel, rpdev) + +/* + * We're allocating buffers of 512 bytes each for communications. The + * number of buffers will be computed from the number of buffers supported + * by the vring, upto a maximum of 512 buffers (256 in each direction). + * + * Each buffer will have 16 bytes for the msg header and 496 bytes for + * the payload. + * + * This will utilize a maximum total space of 256KB for the buffers. + * + * We might also want to add support for user-provided buffers in time. + * This will allow bigger buffer size flexibility, and can also be used + * to achieve zero-copy messaging. + * + * Note that these numbers are purely a decision of this driver - we + * can change this without changing anything in the firmware of the remote + * processor. + */ +#define MAX_RPMSG_NUM_BUFS (512) +#define MAX_RPMSG_BUF_SIZE (512) + +/* + * Local addresses are dynamically allocated on-demand. + * We do not dynamically assign addresses from the low 1024 range, + * in order to reserve that address range for predefined services. + */ +#define RPMSG_RESERVED_ADDRESSES (1024) + +static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept); +static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len); +static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, + u32 dst); +static int virtio_rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, + u32 dst, void *data, int len); +static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len); +static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, + int len, u32 dst); +static int virtio_rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, + u32 dst, void *data, int len); +static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept); +static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp, + struct rpmsg_channel_info *chinfo); + +static const struct rpmsg_endpoint_ops virtio_endpoint_ops = { + .destroy_ept = virtio_rpmsg_destroy_ept, + .send = virtio_rpmsg_send, + .sendto = virtio_rpmsg_sendto, + .send_offchannel = virtio_rpmsg_send_offchannel, + .trysend = virtio_rpmsg_trysend, + .trysendto = virtio_rpmsg_trysendto, + .trysend_offchannel = virtio_rpmsg_trysend_offchannel, + .get_mtu = virtio_rpmsg_get_mtu, +}; + +/** + * rpmsg_sg_init - initialize scatterlist according to cpu address location + * @sg: scatterlist to fill + * @cpu_addr: virtual address of the buffer + * @len: buffer length + * + * An internal function filling scatterlist according to virtual address + * location (in vmalloc or in kernel). + */ +static void +rpmsg_sg_init(struct scatterlist *sg, void *cpu_addr, unsigned int len) +{ + if (is_vmalloc_addr(cpu_addr)) { + sg_init_table(sg, 1); + sg_set_page(sg, vmalloc_to_page(cpu_addr), len, + offset_in_page(cpu_addr)); + } else { + WARN_ON(!virt_addr_valid(cpu_addr)); + sg_init_one(sg, cpu_addr, len); + } +} + +/** + * __ept_release() - deallocate an rpmsg endpoint + * @kref: the ept's reference count + * + * This function deallocates an ept, and is invoked when its @kref refcount + * drops to zero. + * + * Never invoke this function directly! + */ +static void __ept_release(struct kref *kref) +{ + struct rpmsg_endpoint *ept = container_of(kref, struct rpmsg_endpoint, + refcount); + /* + * At this point no one holds a reference to ept anymore, + * so we can directly free it + */ + kfree(ept); +} + +/* for more info, see below documentation of rpmsg_create_ept() */ +static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp, + struct rpmsg_device *rpdev, + rpmsg_rx_cb_t cb, + void *priv, u32 addr) +{ + int id_min, id_max, id; + struct rpmsg_endpoint *ept; + struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev; + + ept = kzalloc(sizeof(*ept), GFP_KERNEL); + if (!ept) + return NULL; + + kref_init(&ept->refcount); + mutex_init(&ept->cb_lock); + + ept->rpdev = rpdev; + ept->cb = cb; + ept->priv = priv; + ept->ops = &virtio_endpoint_ops; + + /* do we need to allocate a local address ? */ + if (addr == RPMSG_ADDR_ANY) { + id_min = RPMSG_RESERVED_ADDRESSES; + id_max = 0; + } else { + id_min = addr; + id_max = addr + 1; + } + + mutex_lock(&vrp->endpoints_lock); + + /* bind the endpoint to an rpmsg address (and allocate one if needed) */ + id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL); + if (id < 0) { + dev_err(dev, "idr_alloc failed: %d\n", id); + goto free_ept; + } + ept->addr = id; + + mutex_unlock(&vrp->endpoints_lock); + + return ept; + +free_ept: + mutex_unlock(&vrp->endpoints_lock); + kref_put(&ept->refcount, __ept_release); + return NULL; +} + +static struct rpmsg_device *virtio_rpmsg_create_channel(struct rpmsg_device *rpdev, + struct rpmsg_channel_info *chinfo) +{ + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + struct virtproc_info *vrp = vch->vrp; + + return __rpmsg_create_channel(vrp, chinfo); +} + +static int virtio_rpmsg_release_channel(struct rpmsg_device *rpdev, + struct rpmsg_channel_info *chinfo) +{ + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + struct virtproc_info *vrp = vch->vrp; + + return rpmsg_unregister_device(&vrp->vdev->dev, chinfo); +} + +static struct rpmsg_endpoint *virtio_rpmsg_create_ept(struct rpmsg_device *rpdev, + rpmsg_rx_cb_t cb, + void *priv, + struct rpmsg_channel_info chinfo) +{ + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + + return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src); +} + +/** + * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint + * @vrp: virtproc which owns this ept + * @ept: endpoing to destroy + * + * An internal function which destroy an ept without assuming it is + * bound to an rpmsg channel. This is needed for handling the internal + * name service endpoint, which isn't bound to an rpmsg channel. + * See also __rpmsg_create_ept(). + */ +static void +__rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept) +{ + /* make sure new inbound messages can't find this ept anymore */ + mutex_lock(&vrp->endpoints_lock); + idr_remove(&vrp->endpoints, ept->addr); + mutex_unlock(&vrp->endpoints_lock); + + /* make sure in-flight inbound messages won't invoke cb anymore */ + mutex_lock(&ept->cb_lock); + ept->cb = NULL; + mutex_unlock(&ept->cb_lock); + + kref_put(&ept->refcount, __ept_release); +} + +static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept) +{ + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev); + + __rpmsg_destroy_ept(vch->vrp, ept); +} + +static int virtio_rpmsg_announce_create(struct rpmsg_device *rpdev) +{ + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + struct virtproc_info *vrp = vch->vrp; + struct device *dev = &rpdev->dev; + int err = 0; + + /* need to tell remote processor's name service about this channel ? */ + if (rpdev->announce && rpdev->ept && + virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { + struct rpmsg_ns_msg nsm; + + strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE); + nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr); + nsm.flags = cpu_to_rpmsg32(rpdev, RPMSG_NS_CREATE); + + err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR); + if (err) + dev_err(dev, "failed to announce service %d\n", err); + } + + return err; +} + +static int virtio_rpmsg_announce_destroy(struct rpmsg_device *rpdev) +{ + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + struct virtproc_info *vrp = vch->vrp; + struct device *dev = &rpdev->dev; + int err = 0; + + /* tell remote processor's name service we're removing this channel */ + if (rpdev->announce && rpdev->ept && + virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { + struct rpmsg_ns_msg nsm; + + strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE); + nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr); + nsm.flags = cpu_to_rpmsg32(rpdev, RPMSG_NS_DESTROY); + + err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR); + if (err) + dev_err(dev, "failed to announce service %d\n", err); + } + + return err; +} + +static const struct rpmsg_device_ops virtio_rpmsg_ops = { + .create_channel = virtio_rpmsg_create_channel, + .release_channel = virtio_rpmsg_release_channel, + .create_ept = virtio_rpmsg_create_ept, + .announce_create = virtio_rpmsg_announce_create, + .announce_destroy = virtio_rpmsg_announce_destroy, +}; + +static void virtio_rpmsg_release_device(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + + kfree(rpdev->driver_override); + kfree(vch); +} + +/* + * create an rpmsg channel using its name and address info. + * this function will be used to create both static and dynamic + * channels. + */ +static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp, + struct rpmsg_channel_info *chinfo) +{ + struct virtio_rpmsg_channel *vch; + struct rpmsg_device *rpdev; + struct device *tmp, *dev = &vrp->vdev->dev; + int ret; + + /* make sure a similar channel doesn't already exist */ + tmp = rpmsg_find_device(dev, chinfo); + if (tmp) { + /* decrement the matched device's refcount back */ + put_device(tmp); + dev_err(dev, "channel %s:%x:%x already exist\n", + chinfo->name, chinfo->src, chinfo->dst); + return NULL; + } + + vch = kzalloc(sizeof(*vch), GFP_KERNEL); + if (!vch) + return NULL; + + /* Link the channel to our vrp */ + vch->vrp = vrp; + + /* Assign public information to the rpmsg_device */ + rpdev = &vch->rpdev; + rpdev->src = chinfo->src; + rpdev->dst = chinfo->dst; + rpdev->ops = &virtio_rpmsg_ops; + rpdev->little_endian = virtio_is_little_endian(vrp->vdev); + + /* + * rpmsg server channels has predefined local address (for now), + * and their existence needs to be announced remotely + */ + rpdev->announce = rpdev->src != RPMSG_ADDR_ANY; + + strncpy(rpdev->id.name, chinfo->name, RPMSG_NAME_SIZE); + + rpdev->dev.parent = &vrp->vdev->dev; + rpdev->dev.release = virtio_rpmsg_release_device; + ret = rpmsg_register_device(rpdev); + if (ret) + return NULL; + + return rpdev; +} + +/* super simple buffer "allocator" that is just enough for now */ +static void *get_a_tx_buf(struct virtproc_info *vrp) +{ + unsigned int len; + void *ret; + + /* support multiple concurrent senders */ + mutex_lock(&vrp->tx_lock); + + /* + * either pick the next unused tx buffer + * (half of our buffers are used for sending messages) + */ + if (vrp->last_sbuf < vrp->num_bufs / 2) + ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++; + /* or recycle a used one */ + else + ret = virtqueue_get_buf(vrp->svq, &len); + + mutex_unlock(&vrp->tx_lock); + + return ret; +} + +/** + * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed + * @vrp: virtual remote processor state + * + * This function is called before a sender is blocked, waiting for + * a tx buffer to become available. + * + * If we already have blocking senders, this function merely increases + * the "sleepers" reference count, and exits. + * + * Otherwise, if this is the first sender to block, we also enable + * virtio's tx callbacks, so we'd be immediately notified when a tx + * buffer is consumed (we rely on virtio's tx callback in order + * to wake up sleeping senders as soon as a tx buffer is used by the + * remote processor). + */ +static void rpmsg_upref_sleepers(struct virtproc_info *vrp) +{ + /* support multiple concurrent senders */ + mutex_lock(&vrp->tx_lock); + + /* are we the first sleeping context waiting for tx buffers ? */ + if (atomic_inc_return(&vrp->sleepers) == 1) + /* enable "tx-complete" interrupts before dozing off */ + virtqueue_enable_cb(vrp->svq); + + mutex_unlock(&vrp->tx_lock); +} + +/** + * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed + * @vrp: virtual remote processor state + * + * This function is called after a sender, that waited for a tx buffer + * to become available, is unblocked. + * + * If we still have blocking senders, this function merely decreases + * the "sleepers" reference count, and exits. + * + * Otherwise, if there are no more blocking senders, we also disable + * virtio's tx callbacks, to avoid the overhead incurred with handling + * those (now redundant) interrupts. + */ +static void rpmsg_downref_sleepers(struct virtproc_info *vrp) +{ + /* support multiple concurrent senders */ + mutex_lock(&vrp->tx_lock); + + /* are we the last sleeping context waiting for tx buffers ? */ + if (atomic_dec_and_test(&vrp->sleepers)) + /* disable "tx-complete" interrupts */ + virtqueue_disable_cb(vrp->svq); + + mutex_unlock(&vrp->tx_lock); +} + +/** + * rpmsg_send_offchannel_raw() - send a message across to the remote processor + * @rpdev: the rpmsg channel + * @src: source address + * @dst: destination address + * @data: payload of message + * @len: length of payload + * @wait: indicates whether caller should block in case no TX buffers available + * + * This function is the base implementation for all of the rpmsg sending API. + * + * It will send @data of length @len to @dst, and say it's from @src. The + * message will be sent to the remote processor which the @rpdev channel + * belongs to. + * + * The message is sent using one of the TX buffers that are available for + * communication with this remote processor. + * + * If @wait is true, the caller will be blocked until either a TX buffer is + * available, or 15 seconds elapses (we don't want callers to + * sleep indefinitely due to misbehaving remote processors), and in that + * case -ERESTARTSYS is returned. The number '15' itself was picked + * arbitrarily; there's little point in asking drivers to provide a timeout + * value themselves. + * + * Otherwise, if @wait is false, and there are no TX buffers available, + * the function will immediately fail, and -ENOMEM will be returned. + * + * Normally drivers shouldn't use this function directly; instead, drivers + * should use the appropriate rpmsg_{try}send{to, _offchannel} API + * (see include/linux/rpmsg.h). + * + * Return: 0 on success and an appropriate error value on failure. + */ +static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev, + u32 src, u32 dst, + void *data, int len, bool wait) +{ + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + struct virtproc_info *vrp = vch->vrp; + struct device *dev = &rpdev->dev; + struct scatterlist sg; + struct rpmsg_hdr *msg; + int err; + + /* bcasting isn't allowed */ + if (src == RPMSG_ADDR_ANY || dst == RPMSG_ADDR_ANY) { + dev_err(dev, "invalid addr (src 0x%x, dst 0x%x)\n", src, dst); + return -EINVAL; + } + + /* + * We currently use fixed-sized buffers, and therefore the payload + * length is limited. + * + * One of the possible improvements here is either to support + * user-provided buffers (and then we can also support zero-copy + * messaging), or to improve the buffer allocator, to support + * variable-length buffer sizes. + */ + if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) { + dev_err(dev, "message is too big (%d)\n", len); + return -EMSGSIZE; + } + + /* grab a buffer */ + msg = get_a_tx_buf(vrp); + if (!msg && !wait) + return -ENOMEM; + + /* no free buffer ? wait for one (but bail after 15 seconds) */ + while (!msg) { + /* enable "tx-complete" interrupts, if not already enabled */ + rpmsg_upref_sleepers(vrp); + + /* + * sleep until a free buffer is available or 15 secs elapse. + * the timeout period is not configurable because there's + * little point in asking drivers to specify that. + * if later this happens to be required, it'd be easy to add. + */ + err = wait_event_interruptible_timeout(vrp->sendq, + (msg = get_a_tx_buf(vrp)), + msecs_to_jiffies(15000)); + + /* disable "tx-complete" interrupts if we're the last sleeper */ + rpmsg_downref_sleepers(vrp); + + /* timeout ? */ + if (!err) { + dev_err(dev, "timeout waiting for a tx buffer\n"); + return -ERESTARTSYS; + } + } + + msg->len = cpu_to_rpmsg16(rpdev, len); + msg->flags = 0; + msg->src = cpu_to_rpmsg32(rpdev, src); + msg->dst = cpu_to_rpmsg32(rpdev, dst); + msg->reserved = 0; + memcpy(msg->data, data, len); + + dev_dbg(dev, "TX From 0x%x, To 0x%x, Len %d, Flags %d, Reserved %d\n", + src, dst, len, msg->flags, msg->reserved); +#if defined(CONFIG_DYNAMIC_DEBUG) + dynamic_hex_dump("rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1, + msg, sizeof(*msg) + len, true); +#endif + + rpmsg_sg_init(&sg, msg, sizeof(*msg) + len); + + mutex_lock(&vrp->tx_lock); + + /* add message to the remote processor's virtqueue */ + err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL); + if (err) { + /* + * need to reclaim the buffer here, otherwise it's lost + * (memory won't leak, but rpmsg won't use it again for TX). + * this will wait for a buffer management overhaul. + */ + dev_err(dev, "virtqueue_add_outbuf failed: %d\n", err); + goto out; + } + + /* tell the remote processor it has a pending message to read */ + virtqueue_kick(vrp->svq); +out: + mutex_unlock(&vrp->tx_lock); + return err; +} + +static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct rpmsg_device *rpdev = ept->rpdev; + u32 src = ept->addr, dst = rpdev->dst; + + return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true); +} + +static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, + u32 dst) +{ + struct rpmsg_device *rpdev = ept->rpdev; + u32 src = ept->addr; + + return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true); +} + +static int virtio_rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, + u32 dst, void *data, int len) +{ + struct rpmsg_device *rpdev = ept->rpdev; + + return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true); +} + +static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct rpmsg_device *rpdev = ept->rpdev; + u32 src = ept->addr, dst = rpdev->dst; + + return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false); +} + +static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, + int len, u32 dst) +{ + struct rpmsg_device *rpdev = ept->rpdev; + u32 src = ept->addr; + + return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false); +} + +static int virtio_rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, + u32 dst, void *data, int len) +{ + struct rpmsg_device *rpdev = ept->rpdev; + + return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false); +} + +static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept) +{ + struct rpmsg_device *rpdev = ept->rpdev; + struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); + + return vch->vrp->buf_size - sizeof(struct rpmsg_hdr); +} + +static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev, + struct rpmsg_hdr *msg, unsigned int len) +{ + struct rpmsg_endpoint *ept; + struct scatterlist sg; + bool little_endian = virtio_is_little_endian(vrp->vdev); + unsigned int msg_len = __rpmsg16_to_cpu(little_endian, msg->len); + int err; + + dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n", + __rpmsg32_to_cpu(little_endian, msg->src), + __rpmsg32_to_cpu(little_endian, msg->dst), msg_len, + __rpmsg16_to_cpu(little_endian, msg->flags), + __rpmsg32_to_cpu(little_endian, msg->reserved)); +#if defined(CONFIG_DYNAMIC_DEBUG) + dynamic_hex_dump("rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1, + msg, sizeof(*msg) + msg_len, true); +#endif + + /* + * We currently use fixed-sized buffers, so trivially sanitize + * the reported payload length. + */ + if (len > vrp->buf_size || + msg_len > (len - sizeof(struct rpmsg_hdr))) { + dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg_len); + return -EINVAL; + } + + /* use the dst addr to fetch the callback of the appropriate user */ + mutex_lock(&vrp->endpoints_lock); + + ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst)); + + /* let's make sure no one deallocates ept while we use it */ + if (ept) + kref_get(&ept->refcount); + + mutex_unlock(&vrp->endpoints_lock); + + if (ept) { + /* make sure ept->cb doesn't go away while we use it */ + mutex_lock(&ept->cb_lock); + + if (ept->cb) + ept->cb(ept->rpdev, msg->data, msg_len, ept->priv, + __rpmsg32_to_cpu(little_endian, msg->src)); + + mutex_unlock(&ept->cb_lock); + + /* farewell, ept, we don't need you anymore */ + kref_put(&ept->refcount, __ept_release); + } else + dev_warn_ratelimited(dev, "msg received with no recipient\n"); + + /* publish the real size of the buffer */ + rpmsg_sg_init(&sg, msg, vrp->buf_size); + + /* add the buffer back to the remote processor's virtqueue */ + err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL); + if (err < 0) { + dev_err(dev, "failed to add a virtqueue buffer: %d\n", err); + return err; + } + + return 0; +} + +/* called when an rx buffer is used, and it's time to digest a message */ +static void rpmsg_recv_done(struct virtqueue *rvq) +{ + struct virtproc_info *vrp = rvq->vdev->priv; + struct device *dev = &rvq->vdev->dev; + struct rpmsg_hdr *msg; + unsigned int len, msgs_received = 0; + int err; + + msg = virtqueue_get_buf(rvq, &len); + if (!msg) { + dev_err(dev, "uhm, incoming signal, but no used buffer ?\n"); + return; + } + + while (msg) { + err = rpmsg_recv_single(vrp, dev, msg, len); + if (err) + break; + + msgs_received++; + + msg = virtqueue_get_buf(rvq, &len); + } + + dev_dbg(dev, "Received %u messages\n", msgs_received); + + /* tell the remote processor we added another available rx buffer */ + if (msgs_received) + virtqueue_kick(vrp->rvq); +} + +/* + * This is invoked whenever the remote processor completed processing + * a TX msg we just sent it, and the buffer is put back to the used ring. + * + * Normally, though, we suppress this "tx complete" interrupt in order to + * avoid the incurred overhead. + */ +static void rpmsg_xmit_done(struct virtqueue *svq) +{ + struct virtproc_info *vrp = svq->vdev->priv; + + dev_dbg(&svq->vdev->dev, "%s\n", __func__); + + /* wake up potential senders that are waiting for a tx buffer */ + wake_up_interruptible(&vrp->sendq); +} + +/* + * Called to expose to user a /dev/rpmsg_ctrlX interface allowing to + * create endpoint-to-endpoint communication without associated RPMsg channel. + * The endpoints are rattached to the ctrldev RPMsg device. + */ +static struct rpmsg_device *rpmsg_virtio_add_ctrl_dev(struct virtio_device *vdev) +{ + struct virtproc_info *vrp = vdev->priv; + struct virtio_rpmsg_channel *vch; + struct rpmsg_device *rpdev_ctrl; + int err = 0; + + vch = kzalloc(sizeof(*vch), GFP_KERNEL); + if (!vch) + return ERR_PTR(-ENOMEM); + + /* Link the channel to the vrp */ + vch->vrp = vrp; + + /* Assign public information to the rpmsg_device */ + rpdev_ctrl = &vch->rpdev; + rpdev_ctrl->ops = &virtio_rpmsg_ops; + + rpdev_ctrl->dev.parent = &vrp->vdev->dev; + rpdev_ctrl->dev.release = virtio_rpmsg_release_device; + rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev); + + err = rpmsg_ctrldev_register_device(rpdev_ctrl); + if (err) { + /* vch will be free in virtio_rpmsg_release_device() */ + return ERR_PTR(err); + } + + return rpdev_ctrl; +} + +static void rpmsg_virtio_del_ctrl_dev(struct rpmsg_device *rpdev_ctrl) +{ + if (!rpdev_ctrl) + return; + device_unregister(&rpdev_ctrl->dev); +} + +static int rpmsg_probe(struct virtio_device *vdev) +{ + vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; + static const char * const names[] = { "input", "output" }; + struct virtqueue *vqs[2]; + struct virtproc_info *vrp; + struct virtio_rpmsg_channel *vch = NULL; + struct rpmsg_device *rpdev_ns, *rpdev_ctrl; + void *bufs_va; + int err = 0, i; + size_t total_buf_space; + bool notify; + + vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); + if (!vrp) + return -ENOMEM; + + vrp->vdev = vdev; + + idr_init(&vrp->endpoints); + mutex_init(&vrp->endpoints_lock); + mutex_init(&vrp->tx_lock); + init_waitqueue_head(&vrp->sendq); + + /* We expect two virtqueues, rx and tx (and in this order) */ + err = virtio_find_vqs(vdev, 2, vqs, vq_cbs, names, NULL); + if (err) + goto free_vrp; + + vrp->rvq = vqs[0]; + vrp->svq = vqs[1]; + + /* we expect symmetric tx/rx vrings */ + WARN_ON(virtqueue_get_vring_size(vrp->rvq) != + virtqueue_get_vring_size(vrp->svq)); + + /* we need less buffers if vrings are small */ + if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2) + vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2; + else + vrp->num_bufs = MAX_RPMSG_NUM_BUFS; + + vrp->buf_size = MAX_RPMSG_BUF_SIZE; + + total_buf_space = vrp->num_bufs * vrp->buf_size; + + /* allocate coherent memory for the buffers */ + bufs_va = dma_alloc_coherent(vdev->dev.parent, + total_buf_space, &vrp->bufs_dma, + GFP_KERNEL); + if (!bufs_va) { + err = -ENOMEM; + goto vqs_del; + } + + dev_dbg(&vdev->dev, "buffers: va %pK, dma %pad\n", + bufs_va, &vrp->bufs_dma); + + /* half of the buffers is dedicated for RX */ + vrp->rbufs = bufs_va; + + /* and half is dedicated for TX */ + vrp->sbufs = bufs_va + total_buf_space / 2; + + /* set up the receive buffers */ + for (i = 0; i < vrp->num_bufs / 2; i++) { + struct scatterlist sg; + void *cpu_addr = vrp->rbufs + i * vrp->buf_size; + + rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size); + + err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr, + GFP_KERNEL); + WARN_ON(err); /* sanity check; this can't really happen */ + } + + /* suppress "tx-complete" interrupts */ + virtqueue_disable_cb(vrp->svq); + + vdev->priv = vrp; + + rpdev_ctrl = rpmsg_virtio_add_ctrl_dev(vdev); + if (IS_ERR(rpdev_ctrl)) { + err = PTR_ERR(rpdev_ctrl); + goto free_coherent; + } + + /* if supported by the remote processor, enable the name service */ + if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { + vch = kzalloc(sizeof(*vch), GFP_KERNEL); + if (!vch) { + err = -ENOMEM; + goto free_ctrldev; + } + + /* Link the channel to our vrp */ + vch->vrp = vrp; + + /* Assign public information to the rpmsg_device */ + rpdev_ns = &vch->rpdev; + rpdev_ns->ops = &virtio_rpmsg_ops; + rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev); + + rpdev_ns->dev.parent = &vrp->vdev->dev; + rpdev_ns->dev.release = virtio_rpmsg_release_device; + + err = rpmsg_ns_register_device(rpdev_ns); + if (err) + /* vch will be free in virtio_rpmsg_release_device() */ + goto free_ctrldev; + } + + /* + * Prepare to kick but don't notify yet - we can't do this before + * device is ready. + */ + notify = virtqueue_kick_prepare(vrp->rvq); + + /* From this point on, we can notify and get callbacks. */ + virtio_device_ready(vdev); + + /* tell the remote processor it can start sending messages */ + /* + * this might be concurrent with callbacks, but we are only + * doing notify, not a full kick here, so that's ok. + */ + if (notify) + virtqueue_notify(vrp->rvq); + + dev_info(&vdev->dev, "rpmsg host is online\n"); + + return 0; + +free_ctrldev: + rpmsg_virtio_del_ctrl_dev(rpdev_ctrl); +free_coherent: + dma_free_coherent(vdev->dev.parent, total_buf_space, + bufs_va, vrp->bufs_dma); +vqs_del: + vdev->config->del_vqs(vrp->vdev); +free_vrp: + kfree(vrp); + return err; +} + +static int rpmsg_remove_device(struct device *dev, void *data) +{ + device_unregister(dev); + + return 0; +} + +static void rpmsg_remove(struct virtio_device *vdev) +{ + struct virtproc_info *vrp = vdev->priv; + size_t total_buf_space = vrp->num_bufs * vrp->buf_size; + int ret; + + virtio_reset_device(vdev); + + ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device); + if (ret) + dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret); + + idr_destroy(&vrp->endpoints); + + vdev->config->del_vqs(vrp->vdev); + + dma_free_coherent(vdev->dev.parent, total_buf_space, + vrp->rbufs, vrp->bufs_dma); + + kfree(vrp); +} + +static struct virtio_device_id id_table[] = { + { VIRTIO_ID_RPMSG, VIRTIO_DEV_ANY_ID }, + { 0 }, +}; + +static unsigned int features[] = { + VIRTIO_RPMSG_F_NS, +}; + +static struct virtio_driver virtio_ipc_driver = { + .feature_table = features, + .feature_table_size = ARRAY_SIZE(features), + .driver.name = KBUILD_MODNAME, + .driver.owner = THIS_MODULE, + .id_table = id_table, + .probe = rpmsg_probe, + .remove = rpmsg_remove, +}; + +static int __init rpmsg_init(void) +{ + int ret; + + ret = register_virtio_driver(&virtio_ipc_driver); + if (ret) + pr_err("failed to register virtio driver: %d\n", ret); + + return ret; +} +subsys_initcall(rpmsg_init); + +static void __exit rpmsg_fini(void) +{ + unregister_virtio_driver(&virtio_ipc_driver); +} +module_exit(rpmsg_fini); + +MODULE_DEVICE_TABLE(virtio, id_table); +MODULE_DESCRIPTION("Virtio-based remote processor messaging bus"); +MODULE_LICENSE("GPL v2"); |