diff options
Diffstat (limited to 'src/spdk/lib/ftl/ftl_io.h')
| -rw-r--r-- | src/spdk/lib/ftl/ftl_io.h | 351 |
1 files changed, 351 insertions, 0 deletions
diff --git a/src/spdk/lib/ftl/ftl_io.h b/src/spdk/lib/ftl/ftl_io.h new file mode 100644 index 000000000..d49dc3de7 --- /dev/null +++ b/src/spdk/lib/ftl/ftl_io.h @@ -0,0 +1,351 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef FTL_IO_H +#define FTL_IO_H + +#include "spdk/stdinc.h" +#include "spdk/nvme.h" +#include "spdk/ftl.h" + +#include "ftl_addr.h" +#include "ftl_trace.h" + +struct spdk_ftl_dev; +struct ftl_band; +struct ftl_batch; +struct ftl_io; + +typedef int (*ftl_md_pack_fn)(struct ftl_band *); +typedef void (*ftl_io_fn)(struct ftl_io *, void *, int); + +/* IO flags */ +enum ftl_io_flags { + /* Indicates whether IO is already initialized */ + FTL_IO_INITIALIZED = (1 << 0), + /* Internal based IO (defrag, metadata etc.) */ + FTL_IO_INTERNAL = (1 << 1), + /* Indicates that the IO should not go through if there's */ + /* already another one scheduled to the same LBA */ + FTL_IO_WEAK = (1 << 2), + /* Indicates that the IO is used for padding */ + FTL_IO_PAD = (1 << 3), + /* The IO operates on metadata */ + FTL_IO_MD = (1 << 4), + /* Using physical instead of logical address */ + FTL_IO_PHYSICAL_MODE = (1 << 5), + /* Indicates that IO contains noncontiguous LBAs */ + FTL_IO_VECTOR_LBA = (1 << 6), + /* The IO is directed to non-volatile cache */ + FTL_IO_CACHE = (1 << 7), + /* Indicates that physical address should be taken from IO struct, */ + /* not assigned by wptr, only works if wptr is also in direct mode */ + FTL_IO_DIRECT_ACCESS = (1 << 8), + /* Bypass the non-volatile cache */ + FTL_IO_BYPASS_CACHE = (1 << 9), +}; + +enum ftl_io_type { + FTL_IO_READ, + FTL_IO_WRITE, + FTL_IO_ERASE, +}; + +#define FTL_IO_MAX_IOVEC 64 + +struct ftl_io_init_opts { + struct spdk_ftl_dev *dev; + + /* IO descriptor */ + struct ftl_io *io; + + /* Parent request */ + struct ftl_io *parent; + + /* Size of IO descriptor */ + size_t size; + + /* IO flags */ + int flags; + + /* IO type */ + enum ftl_io_type type; + + /* Transfer batch, set for IO going through the write buffer */ + struct ftl_batch *batch; + + /* Band to which the IO is directed */ + struct ftl_band *band; + + /* Number of logical blocks */ + size_t num_blocks; + + /* Data */ + struct iovec iovs[FTL_IO_MAX_IOVEC]; + int iovcnt; + + /* Metadata */ + void *md; + + /* Callback's function */ + ftl_io_fn cb_fn; + + /* Callback's context */ + void *cb_ctx; +}; + +struct ftl_io_channel; + +struct ftl_wbuf_entry { + /* IO channel that owns the write bufer entry */ + struct ftl_io_channel *ioch; + /* Data payload (single block) */ + void *payload; + /* Index within the IO channel's wbuf_entries array */ + uint32_t index; + uint32_t io_flags; + /* Points at the band the data is copied from. Only valid for internal + * requests coming from reloc. + */ + struct ftl_band *band; + /* Physical address of that particular block. Valid once the data has + * been written out. + */ + struct ftl_addr addr; + /* Logical block address */ + uint64_t lba; + + /* Trace ID of the requests the entry is part of */ + uint64_t trace; + + /* Indicates that the entry was written out and is still present in the + * L2P table. + */ + bool valid; + /* Lock that protects the entry from being evicted from the L2P */ + pthread_spinlock_t lock; + TAILQ_ENTRY(ftl_wbuf_entry) tailq; +}; + +#define FTL_IO_CHANNEL_INDEX_INVALID ((uint64_t)-1) + +struct ftl_io_channel { + /* Device */ + struct spdk_ftl_dev *dev; + /* IO pool element size */ + size_t elem_size; + /* Index within the IO channel array */ + uint64_t index; + /* IO pool */ + struct spdk_mempool *io_pool; + /* Underlying device IO channel */ + struct spdk_io_channel *base_ioch; + /* Persistent cache IO channel */ + struct spdk_io_channel *cache_ioch; + /* Poller used for completing write requests and retrying IO */ + struct spdk_poller *poller; + /* Write completion queue */ + TAILQ_HEAD(, ftl_io) write_cmpl_queue; + TAILQ_HEAD(, ftl_io) retry_queue; + TAILQ_ENTRY(ftl_io_channel) tailq; + + /* Array of write buffer entries */ + struct ftl_wbuf_entry *wbuf_entries; + /* Write buffer data payload */ + void *wbuf_payload; + /* Number of write buffer entries */ + uint32_t num_entries; + /* Write buffer queues */ + struct spdk_ring *free_queue; + struct spdk_ring *submit_queue; + /* Maximum number of concurrent user writes */ + uint32_t qdepth_limit; + /* Current number of concurrent user writes */ + uint32_t qdepth_current; + /* Means that the IO channel is being flushed */ + bool flush; +}; + +/* General IO descriptor */ +struct ftl_io { + /* Device */ + struct spdk_ftl_dev *dev; + + /* IO channel */ + struct spdk_io_channel *ioch; + + union { + /* LBA table */ + uint64_t *vector; + + /* First LBA */ + uint64_t single; + } lba; + + /* First block address */ + struct ftl_addr addr; + + /* Number of processed blocks */ + size_t pos; + + /* Number of blocks */ + size_t num_blocks; + + /* IO vector pointer */ + struct iovec *iov; + + /* IO vector buffer for internal requests */ + struct iovec iov_buf[FTL_IO_MAX_IOVEC]; + + /* Metadata */ + void *md; + + /* Number of IO vectors */ + size_t iov_cnt; + + /* Position within the iovec */ + size_t iov_pos; + + /* Offset within the iovec (in blocks) */ + size_t iov_off; + + /* Transfer batch (valid only for writes going through the write buffer) */ + struct ftl_batch *batch; + + /* Band this IO is being written to */ + struct ftl_band *band; + + /* Request status */ + int status; + + /* Number of split requests */ + size_t req_cnt; + + /* Callback's function */ + ftl_io_fn cb_fn; + + /* Callback's context */ + void *cb_ctx; + + /* User callback function */ + spdk_ftl_fn user_fn; + + /* Flags */ + int flags; + + /* IO type */ + enum ftl_io_type type; + + /* Done flag */ + bool done; + + /* Parent request */ + struct ftl_io *parent; + /* Child requests list */ + LIST_HEAD(, ftl_io) children; + /* Child list link */ + LIST_ENTRY(ftl_io) child_entry; + /* Children lock */ + pthread_spinlock_t lock; + + /* Trace group id */ + uint64_t trace; + + /* Used by retry and write completion queues */ + TAILQ_ENTRY(ftl_io) ioch_entry; +}; + +/* Metadata IO */ +struct ftl_md_io { + /* Parent IO structure */ + struct ftl_io io; + + /* Serialization/deserialization callback */ + ftl_md_pack_fn pack_fn; + + /* Callback's function */ + ftl_io_fn cb_fn; + + /* Callback's context */ + void *cb_ctx; +}; + +static inline bool +ftl_io_mode_physical(const struct ftl_io *io) +{ + return io->flags & FTL_IO_PHYSICAL_MODE; +} + +static inline bool +ftl_io_mode_logical(const struct ftl_io *io) +{ + return !ftl_io_mode_physical(io); +} + +static inline bool +ftl_io_done(const struct ftl_io *io) +{ + return io->req_cnt == 0 && io->pos == io->num_blocks; +} + +struct ftl_io *ftl_io_alloc(struct spdk_io_channel *ch); +struct ftl_io *ftl_io_alloc_child(struct ftl_io *parent); +void ftl_io_fail(struct ftl_io *io, int status); +void ftl_io_free(struct ftl_io *io); +struct ftl_io *ftl_io_init_internal(const struct ftl_io_init_opts *opts); +void ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb, + void *ctx, int flags, int type); +void ftl_io_clear(struct ftl_io *io); +void ftl_io_inc_req(struct ftl_io *io); +void ftl_io_dec_req(struct ftl_io *io); +struct iovec *ftl_io_iovec(struct ftl_io *io); +uint64_t ftl_io_current_lba(const struct ftl_io *io); +uint64_t ftl_io_get_lba(const struct ftl_io *io, size_t offset); +void ftl_io_advance(struct ftl_io *io, size_t num_blocks); +size_t ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt); +void *ftl_io_iovec_addr(struct ftl_io *io); +size_t ftl_io_iovec_len_left(struct ftl_io *io); +struct ftl_io *ftl_io_wbuf_init(struct spdk_ftl_dev *dev, struct ftl_addr addr, + struct ftl_band *band, struct ftl_batch *batch, ftl_io_fn cb); +struct ftl_io *ftl_io_erase_init(struct ftl_band *band, size_t num_blocks, ftl_io_fn cb); +struct ftl_io *ftl_io_user_init(struct spdk_io_channel *ioch, uint64_t lba, size_t num_blocks, + struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, + void *cb_arg, int type); +void *ftl_io_get_md(const struct ftl_io *io); +void ftl_io_complete(struct ftl_io *io); +void ftl_io_shrink_iovec(struct ftl_io *io, size_t num_blocks); +void ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status); +void ftl_io_reset(struct ftl_io *io); +void ftl_io_call_foreach_child(struct ftl_io *io, int (*callback)(struct ftl_io *)); + +#endif /* FTL_IO_H */ |