diff options
Diffstat (limited to '')
| -rw-r--r-- | plat/nvidia/tegra/drivers/bpmp_ipc/intf.c | 345 | ||||
| -rw-r--r-- | plat/nvidia/tegra/drivers/bpmp_ipc/intf.h | 127 | ||||
| -rw-r--r-- | plat/nvidia/tegra/drivers/bpmp_ipc/ivc.c | 654 | ||||
| -rw-r--r-- | plat/nvidia/tegra/drivers/bpmp_ipc/ivc.h | 50 |
4 files changed, 1176 insertions, 0 deletions
diff --git a/plat/nvidia/tegra/drivers/bpmp_ipc/intf.c b/plat/nvidia/tegra/drivers/bpmp_ipc/intf.c new file mode 100644 index 0000000..2e90d25 --- /dev/null +++ b/plat/nvidia/tegra/drivers/bpmp_ipc/intf.c @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <assert.h> +#include <bpmp_ipc.h> +#include <common/debug.h> +#include <drivers/delay_timer.h> +#include <errno.h> +#include <lib/mmio.h> +#include <lib/utils_def.h> +#include <stdbool.h> +#include <string.h> +#include <tegra_def.h> + +#include "intf.h" +#include "ivc.h" + +/** + * Holds IVC channel data + */ +struct ccplex_bpmp_channel_data { + /* Buffer for incoming data */ + struct frame_data *ib; + + /* Buffer for outgoing data */ + struct frame_data *ob; +}; + +static struct ccplex_bpmp_channel_data s_channel; +static struct ivc ivc_ccplex_bpmp_channel; + +/* + * Helper functions to access the HSP doorbell registers + */ +static inline uint32_t hsp_db_read(uint32_t reg) +{ + return mmio_read_32((uint32_t)(TEGRA_HSP_DBELL_BASE + reg)); +} + +static inline void hsp_db_write(uint32_t reg, uint32_t val) +{ + mmio_write_32((uint32_t)(TEGRA_HSP_DBELL_BASE + reg), val); +} + +/******************************************************************************* + * IVC wrappers for CCPLEX <-> BPMP communication. + ******************************************************************************/ + +static void tegra_bpmp_ring_bpmp_doorbell(void); + +/* + * Get the next frame where data can be written. + */ +static struct frame_data *tegra_bpmp_get_next_out_frame(void) +{ + struct frame_data *frame; + const struct ivc *ch = &ivc_ccplex_bpmp_channel; + + frame = (struct frame_data *)tegra_ivc_write_get_next_frame(ch); + if (frame == NULL) { + ERROR("%s: Error in getting next frame, exiting\n", __func__); + } else { + s_channel.ob = frame; + } + + return frame; +} + +static void tegra_bpmp_signal_slave(void) +{ + (void)tegra_ivc_write_advance(&ivc_ccplex_bpmp_channel); + tegra_bpmp_ring_bpmp_doorbell(); +} + +static int32_t tegra_bpmp_free_master(void) +{ + return tegra_ivc_read_advance(&ivc_ccplex_bpmp_channel); +} + +static bool tegra_bpmp_slave_acked(void) +{ + struct frame_data *frame; + bool ret = true; + + frame = (struct frame_data *)tegra_ivc_read_get_next_frame(&ivc_ccplex_bpmp_channel); + if (frame == NULL) { + ret = false; + } else { + s_channel.ib = frame; + } + + return ret; +} + +static struct frame_data *tegra_bpmp_get_cur_in_frame(void) +{ + return s_channel.ib; +} + +/* + * Enables BPMP to ring CCPlex doorbell + */ +static void tegra_bpmp_enable_ccplex_doorbell(void) +{ + uint32_t reg; + + reg = hsp_db_read(HSP_DBELL_1_ENABLE); + reg |= HSP_MASTER_BPMP_BIT; + hsp_db_write(HSP_DBELL_1_ENABLE, reg); +} + +/* + * CCPlex rings the BPMP doorbell + */ +static void tegra_bpmp_ring_bpmp_doorbell(void) +{ + /* + * Any writes to this register has the same effect, + * uses master ID of the write transaction and set + * corresponding flag. + */ + hsp_db_write(HSP_DBELL_3_TRIGGER, HSP_MASTER_CCPLEX_BIT); +} + +/* + * Returns true if CCPLex can ring BPMP doorbell, otherwise false. + * This also signals that BPMP is up and ready. + */ +static bool tegra_bpmp_can_ccplex_ring_doorbell(void) +{ + uint32_t reg; + + /* check if ccplex can communicate with bpmp */ + reg = hsp_db_read(HSP_DBELL_3_ENABLE); + + return ((reg & HSP_MASTER_CCPLEX_BIT) != 0U); +} + +static int32_t tegra_bpmp_wait_for_slave_ack(void) +{ + uint32_t timeout = TIMEOUT_RESPONSE_FROM_BPMP_US; + + while (!tegra_bpmp_slave_acked() && (timeout != 0U)) { + udelay(1); + timeout--; + }; + + return ((timeout == 0U) ? -ETIMEDOUT : 0); +} + +/* + * Notification from the ivc layer + */ +static void tegra_bpmp_ivc_notify(const struct ivc *ivc) +{ + (void)(ivc); + + tegra_bpmp_ring_bpmp_doorbell(); +} + +/* + * Atomic send/receive API, which means it waits until slave acks + */ +static int32_t tegra_bpmp_ipc_send_req_atomic(uint32_t mrq, void *p_out, + uint32_t size_out, void *p_in, uint32_t size_in) +{ + struct frame_data *frame = tegra_bpmp_get_next_out_frame(); + const struct frame_data *f_in = NULL; + int32_t ret = 0; + void *p_fdata; + + if ((p_out == NULL) || (size_out > IVC_DATA_SZ_BYTES) || + (frame == NULL)) { + ERROR("%s: invalid parameters, exiting\n", __func__); + return -EINVAL; + } + + /* prepare the command frame */ + frame->mrq = mrq; + frame->flags = FLAG_DO_ACK; + p_fdata = frame->data; + (void)memcpy(p_fdata, p_out, (size_t)size_out); + + /* signal the slave */ + tegra_bpmp_signal_slave(); + + /* wait for slave to ack */ + ret = tegra_bpmp_wait_for_slave_ack(); + if (ret < 0) { + ERROR("%s: wait for slave failed (%d)\n", __func__, ret); + return ret; + } + + /* retrieve the response frame */ + if ((size_in <= IVC_DATA_SZ_BYTES) && (p_in != NULL)) { + + f_in = tegra_bpmp_get_cur_in_frame(); + if (f_in != NULL) { + ERROR("Failed to get next input frame!\n"); + } else { + (void)memcpy(p_in, p_fdata, (size_t)size_in); + } + } + + ret = tegra_bpmp_free_master(); + if (ret < 0) { + ERROR("%s: free master failed (%d)\n", __func__, ret); + } + + return ret; +} + +/* + * Initializes the BPMP<--->CCPlex communication path. + */ +int32_t tegra_bpmp_ipc_init(void) +{ + size_t msg_size; + uint32_t frame_size, timeout; + int32_t error = 0; + + /* allow bpmp to ring CCPLEX's doorbell */ + tegra_bpmp_enable_ccplex_doorbell(); + + /* wait for BPMP to actually ring the doorbell */ + timeout = TIMEOUT_RESPONSE_FROM_BPMP_US; + while ((timeout != 0U) && !tegra_bpmp_can_ccplex_ring_doorbell()) { + udelay(1); /* bpmp turn-around time */ + timeout--; + } + + if (timeout == 0U) { + ERROR("%s: BPMP firmware is not ready\n", __func__); + return -ENOTSUP; + } + + INFO("%s: BPMP handshake completed\n", __func__); + + msg_size = tegra_ivc_align(IVC_CMD_SZ_BYTES); + frame_size = (uint32_t)tegra_ivc_total_queue_size(msg_size); + if (frame_size > TEGRA_BPMP_IPC_CH_MAP_SIZE) { + ERROR("%s: carveout size is not sufficient\n", __func__); + return -EINVAL; + } + + error = tegra_ivc_init(&ivc_ccplex_bpmp_channel, + (uint32_t)TEGRA_BPMP_IPC_RX_PHYS_BASE, + (uint32_t)TEGRA_BPMP_IPC_TX_PHYS_BASE, + 1U, frame_size, tegra_bpmp_ivc_notify); + if (error != 0) { + + ERROR("%s: IVC init failed (%d)\n", __func__, error); + + } else { + + /* reset channel */ + tegra_ivc_channel_reset(&ivc_ccplex_bpmp_channel); + + /* wait for notification from BPMP */ + while (tegra_ivc_channel_notified(&ivc_ccplex_bpmp_channel) != 0) { + /* + * Interrupt BPMP with doorbell each time after + * tegra_ivc_channel_notified() returns non zero + * value. + */ + tegra_bpmp_ring_bpmp_doorbell(); + } + + INFO("%s: All communication channels initialized\n", __func__); + } + + return error; +} + +/* Handler to reset a hardware module */ +int32_t tegra_bpmp_ipc_reset_module(uint32_t rst_id) +{ + int32_t ret; + struct mrq_reset_request req = { + .cmd = (uint32_t)CMD_RESET_MODULE, + .reset_id = rst_id + }; + + /* only GPCDMA/XUSB_PADCTL resets are supported */ + assert((rst_id == TEGRA_RESET_ID_XUSB_PADCTL) || + (rst_id == TEGRA_RESET_ID_GPCDMA)); + + ret = tegra_bpmp_ipc_send_req_atomic(MRQ_RESET, &req, + (uint32_t)sizeof(req), NULL, 0); + if (ret != 0) { + ERROR("%s: failed for module %d with error %d\n", __func__, + rst_id, ret); + } + + return ret; +} + +int tegra_bpmp_ipc_enable_clock(uint32_t clk_id) +{ + int ret; + struct mrq_clk_request req; + + /* only SE clocks are supported */ + if (clk_id != TEGRA_CLK_SE) { + return -ENOTSUP; + } + + /* prepare the MRQ_CLK command */ + req.cmd_and_id = make_mrq_clk_cmd(CMD_CLK_ENABLE, clk_id); + + ret = tegra_bpmp_ipc_send_req_atomic(MRQ_CLK, &req, (uint32_t)sizeof(req), + NULL, 0); + if (ret != 0) { + ERROR("%s: failed for module %d with error %d\n", __func__, + clk_id, ret); + } + + return ret; +} + +int tegra_bpmp_ipc_disable_clock(uint32_t clk_id) +{ + int ret; + struct mrq_clk_request req; + + /* only SE clocks are supported */ + if (clk_id != TEGRA_CLK_SE) { + return -ENOTSUP; + } + + /* prepare the MRQ_CLK command */ + req.cmd_and_id = make_mrq_clk_cmd(CMD_CLK_DISABLE, clk_id); + + ret = tegra_bpmp_ipc_send_req_atomic(MRQ_CLK, &req, (uint32_t)sizeof(req), + NULL, 0); + if (ret != 0) { + ERROR("%s: failed for module %d with error %d\n", __func__, + clk_id, ret); + } + + return ret; +} diff --git a/plat/nvidia/tegra/drivers/bpmp_ipc/intf.h b/plat/nvidia/tegra/drivers/bpmp_ipc/intf.h new file mode 100644 index 0000000..d85b906 --- /dev/null +++ b/plat/nvidia/tegra/drivers/bpmp_ipc/intf.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef BPMP_INTF_H +#define BPMP_INTF_H + +/** + * Flags used in IPC req + */ +#define FLAG_DO_ACK (U(1) << 0) +#define FLAG_RING_DOORBELL (U(1) << 1) + +/* Bit 1 is designated for CCPlex in secure world */ +#define HSP_MASTER_CCPLEX_BIT (U(1) << 1) +/* Bit 19 is designated for BPMP in non-secure world */ +#define HSP_MASTER_BPMP_BIT (U(1) << 19) +/* Timeout to receive response from BPMP is 1 sec */ +#define TIMEOUT_RESPONSE_FROM_BPMP_US U(1000000) /* in microseconds */ + +/** + * IVC protocol defines and command/response frame + */ + +/** + * IVC specific defines + */ +#define IVC_CMD_SZ_BYTES U(128) +#define IVC_DATA_SZ_BYTES U(120) + +/** + * Holds frame data for an IPC request + */ +struct frame_data { + /* Identification as to what kind of data is being transmitted */ + uint32_t mrq; + + /* Flags for slave as to how to respond back */ + uint32_t flags; + + /* Actual data being sent */ + uint8_t data[IVC_DATA_SZ_BYTES]; +}; + +/** + * Commands send to the BPMP firmware + */ + +/** + * MRQ command codes + */ +#define MRQ_RESET U(20) +#define MRQ_CLK U(22) + +/** + * Reset sub-commands + */ +#define CMD_RESET_ASSERT U(1) +#define CMD_RESET_DEASSERT U(2) +#define CMD_RESET_MODULE U(3) + +/** + * Used by the sender of an #MRQ_RESET message to request BPMP to + * assert or deassert a given reset line. + */ +struct __attribute__((packed)) mrq_reset_request { + /* reset action to perform (mrq_reset_commands) */ + uint32_t cmd; + /* id of the reset to affected */ + uint32_t reset_id; +}; + +/** + * MRQ_CLK sub-commands + * + */ +enum { + CMD_CLK_GET_RATE = U(1), + CMD_CLK_SET_RATE = U(2), + CMD_CLK_ROUND_RATE = U(3), + CMD_CLK_GET_PARENT = U(4), + CMD_CLK_SET_PARENT = U(5), + CMD_CLK_IS_ENABLED = U(6), + CMD_CLK_ENABLE = U(7), + CMD_CLK_DISABLE = U(8), + CMD_CLK_GET_ALL_INFO = U(14), + CMD_CLK_GET_MAX_CLK_ID = U(15), + CMD_CLK_MAX, +}; + +/** + * Used by the sender of an #MRQ_CLK message to control clocks. The + * clk_request is split into several sub-commands. Some sub-commands + * require no additional data. Others have a sub-command specific + * payload + * + * |sub-command |payload | + * |----------------------------|-----------------------| + * |CMD_CLK_GET_RATE |- | + * |CMD_CLK_SET_RATE |clk_set_rate | + * |CMD_CLK_ROUND_RATE |clk_round_rate | + * |CMD_CLK_GET_PARENT |- | + * |CMD_CLK_SET_PARENT |clk_set_parent | + * |CMD_CLK_IS_ENABLED |- | + * |CMD_CLK_ENABLE |- | + * |CMD_CLK_DISABLE |- | + * |CMD_CLK_GET_ALL_INFO |- | + * |CMD_CLK_GET_MAX_CLK_ID |- | + * + */ +struct mrq_clk_request { + /** + * sub-command and clock id concatenated to 32-bit word. + * - bits[31..24] is the sub-cmd. + * - bits[23..0] is the clock id + */ + uint32_t cmd_and_id; +}; + +/** + * Macro to prepare the MRQ_CLK sub-command + */ +#define make_mrq_clk_cmd(cmd, id) (((cmd) << 24) | (id & 0xFFFFFF)) + +#endif /* BPMP_INTF_H */ diff --git a/plat/nvidia/tegra/drivers/bpmp_ipc/ivc.c b/plat/nvidia/tegra/drivers/bpmp_ipc/ivc.c new file mode 100644 index 0000000..d964fc0 --- /dev/null +++ b/plat/nvidia/tegra/drivers/bpmp_ipc/ivc.c @@ -0,0 +1,654 @@ +/* + * Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch_helpers.h> +#include <assert.h> +#include <common/debug.h> +#include <errno.h> +#include <stdbool.h> +#include <stddef.h> +#include <string.h> + +#include "ivc.h" + +/* + * IVC channel reset protocol. + * + * Each end uses its tx_channel.state to indicate its synchronization state. + */ +enum { + /* + * This value is zero for backwards compatibility with services that + * assume channels to be initially zeroed. Such channels are in an + * initially valid state, but cannot be asynchronously reset, and must + * maintain a valid state at all times. + * + * The transmitting end can enter the established state from the sync or + * ack state when it observes the receiving endpoint in the ack or + * established state, indicating that has cleared the counters in our + * rx_channel. + */ + ivc_state_established = U(0), + + /* + * If an endpoint is observed in the sync state, the remote endpoint is + * allowed to clear the counters it owns asynchronously with respect to + * the current endpoint. Therefore, the current endpoint is no longer + * allowed to communicate. + */ + ivc_state_sync = U(1), + + /* + * When the transmitting end observes the receiving end in the sync + * state, it can clear the w_count and r_count and transition to the ack + * state. If the remote endpoint observes us in the ack state, it can + * return to the established state once it has cleared its counters. + */ + ivc_state_ack = U(2) +}; + +/* + * This structure is divided into two-cache aligned parts, the first is only + * written through the tx_channel pointer, while the second is only written + * through the rx_channel pointer. This delineates ownership of the cache lines, + * which is critical to performance and necessary in non-cache coherent + * implementations. + */ +struct ivc_channel_header { + struct { + /* fields owned by the transmitting end */ + uint32_t w_count; + uint32_t state; + uint32_t w_rsvd[IVC_CHHDR_TX_FIELDS - 2]; + }; + struct { + /* fields owned by the receiving end */ + uint32_t r_count; + uint32_t r_rsvd[IVC_CHHDR_RX_FIELDS - 1]; + }; +}; + +static inline bool ivc_channel_empty(const struct ivc *ivc, + volatile const struct ivc_channel_header *ch) +{ + /* + * This function performs multiple checks on the same values with + * security implications, so sample the counters' current values in + * shared memory to ensure that these checks use the same values. + */ + uint32_t wr_count = ch->w_count; + uint32_t rd_count = ch->r_count; + bool ret = false; + + (void)ivc; + + /* + * Perform an over-full check to prevent denial of service attacks where + * a server could be easily fooled into believing that there's an + * extremely large number of frames ready, since receivers are not + * expected to check for full or over-full conditions. + * + * Although the channel isn't empty, this is an invalid case caused by + * a potentially malicious peer, so returning empty is safer, because it + * gives the impression that the channel has gone silent. + */ + if (((wr_count - rd_count) > ivc->nframes) || (wr_count == rd_count)) { + ret = true; + } + + return ret; +} + +static inline bool ivc_channel_full(const struct ivc *ivc, + volatile const struct ivc_channel_header *ch) +{ + uint32_t wr_count = ch->w_count; + uint32_t rd_count = ch->r_count; + + (void)ivc; + + /* + * Invalid cases where the counters indicate that the queue is over + * capacity also appear full. + */ + return ((wr_count - rd_count) >= ivc->nframes); +} + +static inline uint32_t ivc_channel_avail_count(const struct ivc *ivc, + volatile const struct ivc_channel_header *ch) +{ + uint32_t wr_count = ch->w_count; + uint32_t rd_count = ch->r_count; + + (void)ivc; + + /* + * This function isn't expected to be used in scenarios where an + * over-full situation can lead to denial of service attacks. See the + * comment in ivc_channel_empty() for an explanation about special + * over-full considerations. + */ + return (wr_count - rd_count); +} + +static inline void ivc_advance_tx(struct ivc *ivc) +{ + ivc->tx_channel->w_count++; + + if (ivc->w_pos == (ivc->nframes - (uint32_t)1U)) { + ivc->w_pos = 0U; + } else { + ivc->w_pos++; + } +} + +static inline void ivc_advance_rx(struct ivc *ivc) +{ + ivc->rx_channel->r_count++; + + if (ivc->r_pos == (ivc->nframes - (uint32_t)1U)) { + ivc->r_pos = 0U; + } else { + ivc->r_pos++; + } +} + +static inline int32_t ivc_check_read(const struct ivc *ivc) +{ + /* + * tx_channel->state is set locally, so it is not synchronized with + * state from the remote peer. The remote peer cannot reset its + * transmit counters until we've acknowledged its synchronization + * request, so no additional synchronization is required because an + * asynchronous transition of rx_channel->state to ivc_state_ack is not + * allowed. + */ + if (ivc->tx_channel->state != ivc_state_established) { + return -ECONNRESET; + } + + /* + * Avoid unnecessary invalidations when performing repeated accesses to + * an IVC channel by checking the old queue pointers first. + * Synchronization is only necessary when these pointers indicate empty + * or full. + */ + if (!ivc_channel_empty(ivc, ivc->rx_channel)) { + return 0; + } + + return ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0; +} + +static inline int32_t ivc_check_write(const struct ivc *ivc) +{ + if (ivc->tx_channel->state != ivc_state_established) { + return -ECONNRESET; + } + + if (!ivc_channel_full(ivc, ivc->tx_channel)) { + return 0; + } + + return ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0; +} + +bool tegra_ivc_can_read(const struct ivc *ivc) +{ + return ivc_check_read(ivc) == 0; +} + +bool tegra_ivc_can_write(const struct ivc *ivc) +{ + return ivc_check_write(ivc) == 0; +} + +bool tegra_ivc_tx_empty(const struct ivc *ivc) +{ + return ivc_channel_empty(ivc, ivc->tx_channel); +} + +static inline uintptr_t calc_frame_offset(uint32_t frame_index, + uint32_t frame_size, uint32_t frame_offset) +{ + return ((uintptr_t)frame_index * (uintptr_t)frame_size) + + (uintptr_t)frame_offset; +} + +static void *ivc_frame_pointer(const struct ivc *ivc, + volatile const struct ivc_channel_header *ch, + uint32_t frame) +{ + assert(frame < ivc->nframes); + return (void *)((uintptr_t)(&ch[1]) + + calc_frame_offset(frame, ivc->frame_size, 0)); +} + +int32_t tegra_ivc_read(struct ivc *ivc, void *buf, size_t max_read) +{ + const void *src; + int32_t result; + + if (buf == NULL) { + return -EINVAL; + } + + if (max_read > ivc->frame_size) { + return -E2BIG; + } + + result = ivc_check_read(ivc); + if (result != 0) { + return result; + } + + /* + * Order observation of w_pos potentially indicating new data before + * data read. + */ + dmbish(); + + src = ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos); + + (void)memcpy(buf, src, max_read); + + ivc_advance_rx(ivc); + + /* + * Ensure our write to r_pos occurs before our read from w_pos. + */ + dmbish(); + + /* + * Notify only upon transition from full to non-full. + * The available count can only asynchronously increase, so the + * worst possible side-effect will be a spurious notification. + */ + if (ivc_channel_avail_count(ivc, ivc->rx_channel) == (ivc->nframes - (uint32_t)1U)) { + ivc->notify(ivc); + } + + return (int32_t)max_read; +} + +/* directly peek at the next frame rx'ed */ +void *tegra_ivc_read_get_next_frame(const struct ivc *ivc) +{ + if (ivc_check_read(ivc) != 0) { + return NULL; + } + + /* + * Order observation of w_pos potentially indicating new data before + * data read. + */ + dmbld(); + + return ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos); +} + +int32_t tegra_ivc_read_advance(struct ivc *ivc) +{ + /* + * No read barriers or synchronization here: the caller is expected to + * have already observed the channel non-empty. This check is just to + * catch programming errors. + */ + int32_t result = ivc_check_read(ivc); + if (result != 0) { + return result; + } + + ivc_advance_rx(ivc); + + /* + * Ensure our write to r_pos occurs before our read from w_pos. + */ + dmbish(); + + /* + * Notify only upon transition from full to non-full. + * The available count can only asynchronously increase, so the + * worst possible side-effect will be a spurious notification. + */ + if (ivc_channel_avail_count(ivc, ivc->rx_channel) == (ivc->nframes - (uint32_t)1U)) { + ivc->notify(ivc); + } + + return 0; +} + +int32_t tegra_ivc_write(struct ivc *ivc, const void *buf, size_t size) +{ + void *p; + int32_t result; + + if ((buf == NULL) || (ivc == NULL)) { + return -EINVAL; + } + + if (size > ivc->frame_size) { + return -E2BIG; + } + + result = ivc_check_write(ivc); + if (result != 0) { + return result; + } + + p = ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos); + + (void)memset(p, 0, ivc->frame_size); + (void)memcpy(p, buf, size); + + /* + * Ensure that updated data is visible before the w_pos counter + * indicates that it is ready. + */ + dmbst(); + + ivc_advance_tx(ivc); + + /* + * Ensure our write to w_pos occurs before our read from r_pos. + */ + dmbish(); + + /* + * Notify only upon transition from empty to non-empty. + * The available count can only asynchronously decrease, so the + * worst possible side-effect will be a spurious notification. + */ + if (ivc_channel_avail_count(ivc, ivc->tx_channel) == 1U) { + ivc->notify(ivc); + } + + return (int32_t)size; +} + +/* directly poke at the next frame to be tx'ed */ +void *tegra_ivc_write_get_next_frame(const struct ivc *ivc) +{ + if (ivc_check_write(ivc) != 0) { + return NULL; + } + + return ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos); +} + +/* advance the tx buffer */ +int32_t tegra_ivc_write_advance(struct ivc *ivc) +{ + int32_t result = ivc_check_write(ivc); + + if (result != 0) { + return result; + } + + /* + * Order any possible stores to the frame before update of w_pos. + */ + dmbst(); + + ivc_advance_tx(ivc); + + /* + * Ensure our write to w_pos occurs before our read from r_pos. + */ + dmbish(); + + /* + * Notify only upon transition from empty to non-empty. + * The available count can only asynchronously decrease, so the + * worst possible side-effect will be a spurious notification. + */ + if (ivc_channel_avail_count(ivc, ivc->tx_channel) == (uint32_t)1U) { + ivc->notify(ivc); + } + + return 0; +} + +void tegra_ivc_channel_reset(const struct ivc *ivc) +{ + ivc->tx_channel->state = ivc_state_sync; + ivc->notify(ivc); +} + +/* + * =============================================================== + * IVC State Transition Table - see tegra_ivc_channel_notified() + * =============================================================== + * + * local remote action + * ----- ------ ----------------------------------- + * SYNC EST <none> + * SYNC ACK reset counters; move to EST; notify + * SYNC SYNC reset counters; move to ACK; notify + * ACK EST move to EST; notify + * ACK ACK move to EST; notify + * ACK SYNC reset counters; move to ACK; notify + * EST EST <none> + * EST ACK <none> + * EST SYNC reset counters; move to ACK; notify + * + * =============================================================== + */ +int32_t tegra_ivc_channel_notified(struct ivc *ivc) +{ + uint32_t peer_state; + + /* Copy the receiver's state out of shared memory. */ + peer_state = ivc->rx_channel->state; + + if (peer_state == (uint32_t)ivc_state_sync) { + /* + * Order observation of ivc_state_sync before stores clearing + * tx_channel. + */ + dmbld(); + + /* + * Reset tx_channel counters. The remote end is in the SYNC + * state and won't make progress until we change our state, + * so the counters are not in use at this time. + */ + ivc->tx_channel->w_count = 0U; + ivc->rx_channel->r_count = 0U; + + ivc->w_pos = 0U; + ivc->r_pos = 0U; + + /* + * Ensure that counters appear cleared before new state can be + * observed. + */ + dmbst(); + + /* + * Move to ACK state. We have just cleared our counters, so it + * is now safe for the remote end to start using these values. + */ + ivc->tx_channel->state = ivc_state_ack; + + /* + * Notify remote end to observe state transition. + */ + ivc->notify(ivc); + + } else if ((ivc->tx_channel->state == (uint32_t)ivc_state_sync) && + (peer_state == (uint32_t)ivc_state_ack)) { + /* + * Order observation of ivc_state_sync before stores clearing + * tx_channel. + */ + dmbld(); + + /* + * Reset tx_channel counters. The remote end is in the ACK + * state and won't make progress until we change our state, + * so the counters are not in use at this time. + */ + ivc->tx_channel->w_count = 0U; + ivc->rx_channel->r_count = 0U; + + ivc->w_pos = 0U; + ivc->r_pos = 0U; + + /* + * Ensure that counters appear cleared before new state can be + * observed. + */ + dmbst(); + + /* + * Move to ESTABLISHED state. We know that the remote end has + * already cleared its counters, so it is safe to start + * writing/reading on this channel. + */ + ivc->tx_channel->state = ivc_state_established; + + /* + * Notify remote end to observe state transition. + */ + ivc->notify(ivc); + + } else if (ivc->tx_channel->state == (uint32_t)ivc_state_ack) { + /* + * At this point, we have observed the peer to be in either + * the ACK or ESTABLISHED state. Next, order observation of + * peer state before storing to tx_channel. + */ + dmbld(); + + /* + * Move to ESTABLISHED state. We know that we have previously + * cleared our counters, and we know that the remote end has + * cleared its counters, so it is safe to start writing/reading + * on this channel. + */ + ivc->tx_channel->state = ivc_state_established; + + /* + * Notify remote end to observe state transition. + */ + ivc->notify(ivc); + + } else { + /* + * There is no need to handle any further action. Either the + * channel is already fully established, or we are waiting for + * the remote end to catch up with our current state. Refer + * to the diagram in "IVC State Transition Table" above. + */ + } + + return ((ivc->tx_channel->state == (uint32_t)ivc_state_established) ? 0 : -EAGAIN); +} + +size_t tegra_ivc_align(size_t size) +{ + return (size + (IVC_ALIGN - 1U)) & ~(IVC_ALIGN - 1U); +} + +size_t tegra_ivc_total_queue_size(size_t queue_size) +{ + if ((queue_size & (IVC_ALIGN - 1U)) != 0U) { + ERROR("queue_size (%d) must be %d-byte aligned\n", + (int32_t)queue_size, IVC_ALIGN); + return 0; + } + return queue_size + sizeof(struct ivc_channel_header); +} + +static int32_t check_ivc_params(uintptr_t queue_base1, uintptr_t queue_base2, + uint32_t nframes, uint32_t frame_size) +{ + assert((offsetof(struct ivc_channel_header, w_count) + & (IVC_ALIGN - 1U)) == 0U); + assert((offsetof(struct ivc_channel_header, r_count) + & (IVC_ALIGN - 1U)) == 0U); + assert((sizeof(struct ivc_channel_header) & (IVC_ALIGN - 1U)) == 0U); + + if (((uint64_t)nframes * (uint64_t)frame_size) >= 0x100000000ULL) { + ERROR("nframes * frame_size overflows\n"); + return -EINVAL; + } + + /* + * The headers must at least be aligned enough for counters + * to be accessed atomically. + */ + if ((queue_base1 & (IVC_ALIGN - 1U)) != 0U) { + ERROR("ivc channel start not aligned: %lx\n", queue_base1); + return -EINVAL; + } + if ((queue_base2 & (IVC_ALIGN - 1U)) != 0U) { + ERROR("ivc channel start not aligned: %lx\n", queue_base2); + return -EINVAL; + } + + if ((frame_size & (IVC_ALIGN - 1U)) != 0U) { + ERROR("frame size not adequately aligned: %u\n", + frame_size); + return -EINVAL; + } + + if (queue_base1 < queue_base2) { + if ((queue_base1 + ((uint64_t)frame_size * nframes)) > queue_base2) { + ERROR("queue regions overlap: %lx + %x, %x\n", + queue_base1, frame_size, + frame_size * nframes); + return -EINVAL; + } + } else { + if ((queue_base2 + ((uint64_t)frame_size * nframes)) > queue_base1) { + ERROR("queue regions overlap: %lx + %x, %x\n", + queue_base2, frame_size, + frame_size * nframes); + return -EINVAL; + } + } + + return 0; +} + +int32_t tegra_ivc_init(struct ivc *ivc, uintptr_t rx_base, uintptr_t tx_base, + uint32_t nframes, uint32_t frame_size, + ivc_notify_function notify) +{ + int32_t result; + + /* sanity check input params */ + if ((ivc == NULL) || (notify == NULL)) { + return -EINVAL; + } + + result = check_ivc_params(rx_base, tx_base, nframes, frame_size); + if (result != 0) { + return result; + } + + /* + * All sizes that can be returned by communication functions should + * fit in a 32-bit integer. + */ + if (frame_size > (1u << 31)) { + return -E2BIG; + } + + ivc->rx_channel = (struct ivc_channel_header *)rx_base; + ivc->tx_channel = (struct ivc_channel_header *)tx_base; + ivc->notify = notify; + ivc->frame_size = frame_size; + ivc->nframes = nframes; + ivc->w_pos = 0U; + ivc->r_pos = 0U; + + INFO("%s: done\n", __func__); + + return 0; +} diff --git a/plat/nvidia/tegra/drivers/bpmp_ipc/ivc.h b/plat/nvidia/tegra/drivers/bpmp_ipc/ivc.h new file mode 100644 index 0000000..1b31821 --- /dev/null +++ b/plat/nvidia/tegra/drivers/bpmp_ipc/ivc.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2017-2020, NVIDIA Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef BPMP_IVC_H +#define BPMP_IVC_H + +#include <lib/utils_def.h> +#include <stdint.h> +#include <stddef.h> + +#define IVC_ALIGN U(64) +#define IVC_CHHDR_TX_FIELDS U(16) +#define IVC_CHHDR_RX_FIELDS U(16) + +struct ivc_channel_header; + +struct ivc { + struct ivc_channel_header *rx_channel; + struct ivc_channel_header *tx_channel; + uint32_t w_pos; + uint32_t r_pos; + void (*notify)(const struct ivc *); + uint32_t nframes; + uint32_t frame_size; +}; + +/* callback handler for notify on receiving a response */ +typedef void (* ivc_notify_function)(const struct ivc *); + +int32_t tegra_ivc_init(struct ivc *ivc, uintptr_t rx_base, uintptr_t tx_base, + uint32_t nframes, uint32_t frame_size, + ivc_notify_function notify); +size_t tegra_ivc_total_queue_size(size_t queue_size); +size_t tegra_ivc_align(size_t size); +int32_t tegra_ivc_channel_notified(struct ivc *ivc); +void tegra_ivc_channel_reset(const struct ivc *ivc); +int32_t tegra_ivc_write_advance(struct ivc *ivc); +void *tegra_ivc_write_get_next_frame(const struct ivc *ivc); +int32_t tegra_ivc_write(struct ivc *ivc, const void *buf, size_t size); +int32_t tegra_ivc_read_advance(struct ivc *ivc); +void *tegra_ivc_read_get_next_frame(const struct ivc *ivc); +int32_t tegra_ivc_read(struct ivc *ivc, void *buf, size_t max_read); +bool tegra_ivc_tx_empty(const struct ivc *ivc); +bool tegra_ivc_can_write(const struct ivc *ivc); +bool tegra_ivc_can_read(const struct ivc *ivc); + +#endif /* BPMP_IVC_H */ |