diff options
Diffstat (limited to 'src/spdk/lib/ftl/ftl_core.h')
| -rw-r--r-- | src/spdk/lib/ftl/ftl_core.h | 552 |
1 files changed, 552 insertions, 0 deletions
diff --git a/src/spdk/lib/ftl/ftl_core.h b/src/spdk/lib/ftl/ftl_core.h new file mode 100644 index 000000000..b782ba731 --- /dev/null +++ b/src/spdk/lib/ftl/ftl_core.h @@ -0,0 +1,552 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef FTL_CORE_H +#define FTL_CORE_H + +#include "spdk/stdinc.h" +#include "spdk/uuid.h" +#include "spdk/thread.h" +#include "spdk/util.h" +#include "spdk_internal/log.h" +#include "spdk/likely.h" +#include "spdk/queue.h" +#include "spdk/ftl.h" +#include "spdk/bdev.h" +#include "spdk/bdev_zone.h" + +#include "ftl_addr.h" +#include "ftl_io.h" +#include "ftl_trace.h" + +#ifdef SPDK_CONFIG_PMDK +#include "libpmem.h" +#endif /* SPDK_CONFIG_PMDK */ + +struct spdk_ftl_dev; +struct ftl_band; +struct ftl_zone; +struct ftl_io; +struct ftl_restore; +struct ftl_wptr; +struct ftl_flush; +struct ftl_reloc; +struct ftl_anm_event; +struct ftl_band_flush; + +struct ftl_stats { + /* Number of writes scheduled directly by the user */ + uint64_t write_user; + + /* Total number of writes */ + uint64_t write_total; + + /* Traces */ + struct ftl_trace trace; + + /* Number of limits applied */ + uint64_t limits[SPDK_FTL_LIMIT_MAX]; +}; + +struct ftl_global_md { + /* Device instance */ + struct spdk_uuid uuid; + /* Size of the l2p table */ + uint64_t num_lbas; +}; + +struct ftl_nv_cache { + /* Write buffer cache bdev */ + struct spdk_bdev_desc *bdev_desc; + /* Write pointer */ + uint64_t current_addr; + /* Number of available blocks left */ + uint64_t num_available; + /* Maximum number of blocks */ + uint64_t num_data_blocks; + /* + * Phase of the current cycle of writes. Each time whole cache area is filled, the phase is + * advanced. Current phase is saved in every IO's metadata, as well as in the header saved + * in the first sector. By looking at the phase of each block, it's possible to find the + * oldest block and replay the order of the writes when recovering the data from the cache. + */ + unsigned int phase; + /* Indicates that the data can be written to the cache */ + bool ready; + /* Metadata pool */ + struct spdk_mempool *md_pool; + /* DMA buffer for writing the header */ + void *dma_buf; + /* Cache lock */ + pthread_spinlock_t lock; +}; + +struct ftl_batch { + /* Queue of write buffer entries, can reach up to xfer_size entries */ + TAILQ_HEAD(, ftl_wbuf_entry) entries; + /* Number of entries in the queue above */ + uint32_t num_entries; + /* Index within spdk_ftl_dev.batch_array */ + uint32_t index; + struct iovec *iov; + void *metadata; + TAILQ_ENTRY(ftl_batch) tailq; +}; + +struct spdk_ftl_dev { + /* Device instance */ + struct spdk_uuid uuid; + /* Device name */ + char *name; + /* Configuration */ + struct spdk_ftl_conf conf; + + /* Indicates the device is fully initialized */ + int initialized; + /* Indicates the device is about to be stopped */ + int halt; + /* Indicates the device is about to start stopping - use to handle multiple stop request */ + bool halt_started; + + /* Underlying device */ + struct spdk_bdev_desc *base_bdev_desc; + + /* Non-volatile write buffer cache */ + struct ftl_nv_cache nv_cache; + + /* LBA map memory pool */ + struct spdk_mempool *lba_pool; + + /* LBA map requests pool */ + struct spdk_mempool *lba_request_pool; + + /* Media management events pool */ + struct spdk_mempool *media_events_pool; + + /* Statistics */ + struct ftl_stats stats; + + /* Current sequence number */ + uint64_t seq; + + /* Array of bands */ + struct ftl_band *bands; + /* Number of operational bands */ + size_t num_bands; + /* Next write band */ + struct ftl_band *next_band; + /* Free band list */ + LIST_HEAD(, ftl_band) free_bands; + /* Closed bands list */ + LIST_HEAD(, ftl_band) shut_bands; + /* Number of free bands */ + size_t num_free; + + /* List of write pointers */ + LIST_HEAD(, ftl_wptr) wptr_list; + + /* Logical -> physical table */ + void *l2p; + /* Size of the l2p table */ + uint64_t num_lbas; + /* Size of pages mmapped for l2p, valid only for mapping on persistent memory */ + size_t l2p_pmem_len; + + /* Address size */ + size_t addr_len; + + /* Flush list */ + LIST_HEAD(, ftl_flush) flush_list; + /* List of band flush requests */ + LIST_HEAD(, ftl_band_flush) band_flush_list; + + /* Device specific md buffer */ + struct ftl_global_md global_md; + + /* Metadata size */ + size_t md_size; + void *md_buf; + + /* Transfer unit size */ + size_t xfer_size; + + /* Current user write limit */ + int limit; + + /* Inflight IO operations */ + uint32_t num_inflight; + + /* Manages data relocation */ + struct ftl_reloc *reloc; + + /* Thread on which the poller is running */ + struct spdk_thread *core_thread; + /* IO channel */ + struct spdk_io_channel *ioch; + /* Poller */ + struct spdk_poller *core_poller; + + /* IO channel array provides means for retrieving write buffer entries + * from their address stored in L2P. The address is divided into two + * parts - IO channel offset poining at specific IO channel (within this + * array) and entry offset pointing at specific entry within that IO + * channel. + */ + struct ftl_io_channel **ioch_array; + TAILQ_HEAD(, ftl_io_channel) ioch_queue; + uint64_t num_io_channels; + /* Value required to shift address of a write buffer entry to retrieve + * the IO channel it's part of. The other part of the address describes + * the offset of an entry within the IO channel's entry array. + */ + uint64_t ioch_shift; + + /* Write buffer batches */ +#define FTL_BATCH_COUNT 4096 + struct ftl_batch batch_array[FTL_BATCH_COUNT]; + /* Iovec buffer used by batches */ + struct iovec *iov_buf; + /* Batch currently being filled */ + struct ftl_batch *current_batch; + /* Full and ready to be sent batches. A batch is put on this queue in + * case it's already filled, but cannot be sent. + */ + TAILQ_HEAD(, ftl_batch) pending_batches; + TAILQ_HEAD(, ftl_batch) free_batches; + + /* Devices' list */ + STAILQ_ENTRY(spdk_ftl_dev) stailq; +}; + +struct ftl_nv_cache_header { + /* Version of the header */ + uint32_t version; + /* UUID of the FTL device */ + struct spdk_uuid uuid; + /* Size of the non-volatile cache (in blocks) */ + uint64_t size; + /* Contains the next address to be written after clean shutdown, invalid LBA otherwise */ + uint64_t current_addr; + /* Current phase */ + uint8_t phase; + /* Checksum of the header, needs to be last element */ + uint32_t checksum; +} __attribute__((packed)); + +struct ftl_media_event { + /* Owner */ + struct spdk_ftl_dev *dev; + /* Media event */ + struct spdk_bdev_media_event event; +}; + +typedef void (*ftl_restore_fn)(struct ftl_restore *, int, void *cb_arg); + +void ftl_apply_limits(struct spdk_ftl_dev *dev); +void ftl_io_read(struct ftl_io *io); +void ftl_io_write(struct ftl_io *io); +int ftl_flush_wbuf(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); +int ftl_current_limit(const struct spdk_ftl_dev *dev); +int ftl_invalidate_addr(struct spdk_ftl_dev *dev, struct ftl_addr addr); +int ftl_task_core(void *ctx); +int ftl_task_read(void *ctx); +void ftl_process_anm_event(struct ftl_anm_event *event); +size_t ftl_tail_md_num_blocks(const struct spdk_ftl_dev *dev); +size_t ftl_tail_md_hdr_num_blocks(void); +size_t ftl_vld_map_num_blocks(const struct spdk_ftl_dev *dev); +size_t ftl_lba_map_num_blocks(const struct spdk_ftl_dev *dev); +size_t ftl_head_md_num_blocks(const struct spdk_ftl_dev *dev); +int ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb, void *cb_arg); +int ftl_restore_device(struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg); +void ftl_restore_nv_cache(struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg); +int ftl_band_set_direct_access(struct ftl_band *band, bool access); +bool ftl_addr_is_written(struct ftl_band *band, struct ftl_addr addr); +int ftl_flush_active_bands(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); +int ftl_nv_cache_write_header(struct ftl_nv_cache *nv_cache, bool shutdown, + spdk_bdev_io_completion_cb cb_fn, void *cb_arg); +int ftl_nv_cache_scrub(struct ftl_nv_cache *nv_cache, spdk_bdev_io_completion_cb cb_fn, + void *cb_arg); +void ftl_get_media_events(struct spdk_ftl_dev *dev); +int ftl_io_channel_poll(void *arg); +void ftl_evict_cache_entry(struct spdk_ftl_dev *dev, struct ftl_wbuf_entry *entry); +struct spdk_io_channel *ftl_get_io_channel(const struct spdk_ftl_dev *dev); +struct ftl_io_channel *ftl_io_channel_get_ctx(struct spdk_io_channel *ioch); + + +#define ftl_to_addr(address) \ + (struct ftl_addr) { .offset = (uint64_t)(address) } + +#define ftl_to_addr_packed(address) \ + (struct ftl_addr) { .pack.offset = (uint32_t)(address) } + +static inline struct spdk_thread * +ftl_get_core_thread(const struct spdk_ftl_dev *dev) +{ + return dev->core_thread; +} + +static inline size_t +ftl_get_num_bands(const struct spdk_ftl_dev *dev) +{ + return dev->num_bands; +} + +static inline size_t +ftl_get_num_punits(const struct spdk_ftl_dev *dev) +{ + return spdk_bdev_get_optimal_open_zones(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); +} + +static inline size_t +ftl_get_num_zones(const struct spdk_ftl_dev *dev) +{ + return ftl_get_num_bands(dev) * ftl_get_num_punits(dev); +} + +static inline size_t +ftl_get_num_blocks_in_zone(const struct spdk_ftl_dev *dev) +{ + return spdk_bdev_get_zone_size(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); +} + +static inline uint64_t +ftl_get_num_blocks_in_band(const struct spdk_ftl_dev *dev) +{ + return ftl_get_num_punits(dev) * ftl_get_num_blocks_in_zone(dev); +} + +static inline uint64_t +ftl_addr_get_zone_slba(const struct spdk_ftl_dev *dev, struct ftl_addr addr) +{ + return addr.offset -= (addr.offset % ftl_get_num_blocks_in_zone(dev)); +} + +static inline uint64_t +ftl_addr_get_band(const struct spdk_ftl_dev *dev, struct ftl_addr addr) +{ + return addr.offset / ftl_get_num_blocks_in_band(dev); +} + +static inline uint64_t +ftl_addr_get_punit(const struct spdk_ftl_dev *dev, struct ftl_addr addr) +{ + return (addr.offset / ftl_get_num_blocks_in_zone(dev)) % ftl_get_num_punits(dev); +} + +static inline uint64_t +ftl_addr_get_zone_offset(const struct spdk_ftl_dev *dev, struct ftl_addr addr) +{ + return addr.offset % ftl_get_num_blocks_in_zone(dev); +} + +static inline size_t +ftl_vld_map_size(const struct spdk_ftl_dev *dev) +{ + return (size_t)spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), CHAR_BIT); +} + +static inline int +ftl_addr_packed(const struct spdk_ftl_dev *dev) +{ + return dev->addr_len < 32; +} + +static inline void +ftl_l2p_lba_persist(const struct spdk_ftl_dev *dev, uint64_t lba) +{ +#ifdef SPDK_CONFIG_PMDK + size_t ftl_addr_size = ftl_addr_packed(dev) ? 4 : 8; + pmem_persist((char *)dev->l2p + (lba * ftl_addr_size), ftl_addr_size); +#else /* SPDK_CONFIG_PMDK */ + SPDK_ERRLOG("Libpmem not available, cannot flush l2p to pmem\n"); + assert(0); +#endif /* SPDK_CONFIG_PMDK */ +} + +static inline int +ftl_addr_invalid(struct ftl_addr addr) +{ + return addr.offset == ftl_to_addr(FTL_ADDR_INVALID).offset; +} + +static inline int +ftl_addr_cached(struct ftl_addr addr) +{ + return !ftl_addr_invalid(addr) && addr.cached; +} + +static inline struct ftl_addr +ftl_addr_to_packed(const struct spdk_ftl_dev *dev, struct ftl_addr addr) +{ + struct ftl_addr p = {}; + + if (ftl_addr_invalid(addr)) { + p = ftl_to_addr_packed(FTL_ADDR_INVALID); + } else if (ftl_addr_cached(addr)) { + p.pack.cached = 1; + p.pack.cache_offset = (uint32_t) addr.cache_offset; + } else { + p.pack.offset = (uint32_t) addr.offset; + } + + return p; +} + +static inline struct ftl_addr +ftl_addr_from_packed(const struct spdk_ftl_dev *dev, struct ftl_addr p) +{ + struct ftl_addr addr = {}; + + if (p.pack.offset == (uint32_t)FTL_ADDR_INVALID) { + addr = ftl_to_addr(FTL_ADDR_INVALID); + } else if (p.pack.cached) { + addr.cached = 1; + addr.cache_offset = p.pack.cache_offset; + } else { + addr = p; + } + + return addr; +} + +#define _ftl_l2p_set(l2p, off, val, bits) \ + __atomic_store_n(((uint##bits##_t *)(l2p)) + (off), val, __ATOMIC_SEQ_CST) + +#define _ftl_l2p_set32(l2p, off, val) \ + _ftl_l2p_set(l2p, off, val, 32) + +#define _ftl_l2p_set64(l2p, off, val) \ + _ftl_l2p_set(l2p, off, val, 64) + +#define _ftl_l2p_get(l2p, off, bits) \ + __atomic_load_n(((uint##bits##_t *)(l2p)) + (off), __ATOMIC_SEQ_CST) + +#define _ftl_l2p_get32(l2p, off) \ + _ftl_l2p_get(l2p, off, 32) + +#define _ftl_l2p_get64(l2p, off) \ + _ftl_l2p_get(l2p, off, 64) + +#define ftl_addr_cmp(p1, p2) \ + ((p1).offset == (p2).offset) + +static inline void +ftl_l2p_set(struct spdk_ftl_dev *dev, uint64_t lba, struct ftl_addr addr) +{ + assert(dev->num_lbas > lba); + + if (ftl_addr_packed(dev)) { + _ftl_l2p_set32(dev->l2p, lba, ftl_addr_to_packed(dev, addr).offset); + } else { + _ftl_l2p_set64(dev->l2p, lba, addr.offset); + } + + if (dev->l2p_pmem_len != 0) { + ftl_l2p_lba_persist(dev, lba); + } +} + +static inline struct ftl_addr +ftl_l2p_get(struct spdk_ftl_dev *dev, uint64_t lba) +{ + assert(dev->num_lbas > lba); + + if (ftl_addr_packed(dev)) { + return ftl_addr_from_packed(dev, ftl_to_addr_packed( + _ftl_l2p_get32(dev->l2p, lba))); + } else { + return ftl_to_addr(_ftl_l2p_get64(dev->l2p, lba)); + } +} + +static inline bool +ftl_dev_has_nv_cache(const struct spdk_ftl_dev *dev) +{ + return dev->nv_cache.bdev_desc != NULL; +} + +#define FTL_NV_CACHE_HEADER_VERSION (1) +#define FTL_NV_CACHE_DATA_OFFSET (1) +#define FTL_NV_CACHE_PHASE_OFFSET (62) +#define FTL_NV_CACHE_PHASE_COUNT (4) +#define FTL_NV_CACHE_PHASE_MASK (3ULL << FTL_NV_CACHE_PHASE_OFFSET) +#define FTL_NV_CACHE_LBA_INVALID (FTL_LBA_INVALID & ~FTL_NV_CACHE_PHASE_MASK) + +static inline bool +ftl_nv_cache_phase_is_valid(unsigned int phase) +{ + return phase > 0 && phase <= 3; +} + +static inline unsigned int +ftl_nv_cache_next_phase(unsigned int current) +{ + static const unsigned int phases[] = { 0, 2, 3, 1 }; + assert(ftl_nv_cache_phase_is_valid(current)); + return phases[current]; +} + +static inline unsigned int +ftl_nv_cache_prev_phase(unsigned int current) +{ + static const unsigned int phases[] = { 0, 3, 1, 2 }; + assert(ftl_nv_cache_phase_is_valid(current)); + return phases[current]; +} + +static inline uint64_t +ftl_nv_cache_pack_lba(uint64_t lba, unsigned int phase) +{ + assert(ftl_nv_cache_phase_is_valid(phase)); + return (lba & ~FTL_NV_CACHE_PHASE_MASK) | ((uint64_t)phase << FTL_NV_CACHE_PHASE_OFFSET); +} + +static inline void +ftl_nv_cache_unpack_lba(uint64_t in_lba, uint64_t *out_lba, unsigned int *phase) +{ + *out_lba = in_lba & ~FTL_NV_CACHE_PHASE_MASK; + *phase = (in_lba & FTL_NV_CACHE_PHASE_MASK) >> FTL_NV_CACHE_PHASE_OFFSET; + + /* If the phase is invalid the block wasn't written yet, so treat the LBA as invalid too */ + if (!ftl_nv_cache_phase_is_valid(*phase) || *out_lba == FTL_NV_CACHE_LBA_INVALID) { + *out_lba = FTL_LBA_INVALID; + } +} + +static inline bool +ftl_is_append_supported(const struct spdk_ftl_dev *dev) +{ + return dev->conf.use_append; +} + +#endif /* FTL_CORE_H */ |