summaryrefslogtreecommitdiffstats
path: root/drivers/md/dm-bio-prison-v2.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/md/dm-bio-prison-v2.c')
-rw-r--r--drivers/md/dm-bio-prison-v2.c365
1 files changed, 365 insertions, 0 deletions
diff --git a/drivers/md/dm-bio-prison-v2.c b/drivers/md/dm-bio-prison-v2.c
new file mode 100644
index 0000000000..fd852981ef
--- /dev/null
+++ b/drivers/md/dm-bio-prison-v2.c
@@ -0,0 +1,365 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012-2017 Red Hat, Inc.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-bio-prison-v2.h"
+
+#include <linux/spinlock.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/rwsem.h>
+
+/*----------------------------------------------------------------*/
+
+#define MIN_CELLS 1024
+
+struct dm_bio_prison_v2 {
+ struct workqueue_struct *wq;
+
+ spinlock_t lock;
+ struct rb_root cells;
+ mempool_t cell_pool;
+};
+
+static struct kmem_cache *_cell_cache;
+
+/*----------------------------------------------------------------*/
+
+/*
+ * @nr_cells should be the number of cells you want in use _concurrently_.
+ * Don't confuse it with the number of distinct keys.
+ */
+struct dm_bio_prison_v2 *dm_bio_prison_create_v2(struct workqueue_struct *wq)
+{
+ struct dm_bio_prison_v2 *prison = kzalloc(sizeof(*prison), GFP_KERNEL);
+ int ret;
+
+ if (!prison)
+ return NULL;
+
+ prison->wq = wq;
+ spin_lock_init(&prison->lock);
+
+ ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
+ if (ret) {
+ kfree(prison);
+ return NULL;
+ }
+
+ prison->cells = RB_ROOT;
+
+ return prison;
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_create_v2);
+
+void dm_bio_prison_destroy_v2(struct dm_bio_prison_v2 *prison)
+{
+ mempool_exit(&prison->cell_pool);
+ kfree(prison);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_destroy_v2);
+
+struct dm_bio_prison_cell_v2 *dm_bio_prison_alloc_cell_v2(struct dm_bio_prison_v2 *prison, gfp_t gfp)
+{
+ return mempool_alloc(&prison->cell_pool, gfp);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell_v2);
+
+void dm_bio_prison_free_cell_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell)
+{
+ mempool_free(cell, &prison->cell_pool);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell_v2);
+
+static void __setup_new_cell(struct dm_cell_key_v2 *key,
+ struct dm_bio_prison_cell_v2 *cell)
+{
+ memset(cell, 0, sizeof(*cell));
+ memcpy(&cell->key, key, sizeof(cell->key));
+ bio_list_init(&cell->bios);
+}
+
+static int cmp_keys(struct dm_cell_key_v2 *lhs,
+ struct dm_cell_key_v2 *rhs)
+{
+ if (lhs->virtual < rhs->virtual)
+ return -1;
+
+ if (lhs->virtual > rhs->virtual)
+ return 1;
+
+ if (lhs->dev < rhs->dev)
+ return -1;
+
+ if (lhs->dev > rhs->dev)
+ return 1;
+
+ if (lhs->block_end <= rhs->block_begin)
+ return -1;
+
+ if (lhs->block_begin >= rhs->block_end)
+ return 1;
+
+ return 0;
+}
+
+/*
+ * Returns true if node found, otherwise it inserts a new one.
+ */
+static bool __find_or_insert(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **result)
+{
+ int r;
+ struct rb_node **new = &prison->cells.rb_node, *parent = NULL;
+
+ while (*new) {
+ struct dm_bio_prison_cell_v2 *cell =
+ rb_entry(*new, struct dm_bio_prison_cell_v2, node);
+
+ r = cmp_keys(key, &cell->key);
+
+ parent = *new;
+ if (r < 0)
+ new = &((*new)->rb_left);
+
+ else if (r > 0)
+ new = &((*new)->rb_right);
+
+ else {
+ *result = cell;
+ return true;
+ }
+ }
+
+ __setup_new_cell(key, cell_prealloc);
+ *result = cell_prealloc;
+ rb_link_node(&cell_prealloc->node, parent, new);
+ rb_insert_color(&cell_prealloc->node, &prison->cells);
+
+ return false;
+}
+
+static bool __get(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned int lock_level,
+ struct bio *inmate,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell)
+{
+ if (__find_or_insert(prison, key, cell_prealloc, cell)) {
+ if ((*cell)->exclusive_lock) {
+ if (lock_level <= (*cell)->exclusive_level) {
+ bio_list_add(&(*cell)->bios, inmate);
+ return false;
+ }
+ }
+
+ (*cell)->shared_count++;
+
+ } else
+ (*cell)->shared_count = 1;
+
+ return true;
+}
+
+bool dm_cell_get_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned int lock_level,
+ struct bio *inmate,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell_result)
+{
+ int r;
+
+ spin_lock_irq(&prison->lock);
+ r = __get(prison, key, lock_level, inmate, cell_prealloc, cell_result);
+ spin_unlock_irq(&prison->lock);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_get_v2);
+
+static bool __put(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell)
+{
+ BUG_ON(!cell->shared_count);
+ cell->shared_count--;
+
+ // FIXME: shared locks granted above the lock level could starve this
+ if (!cell->shared_count) {
+ if (cell->exclusive_lock) {
+ if (cell->quiesce_continuation) {
+ queue_work(prison->wq, cell->quiesce_continuation);
+ cell->quiesce_continuation = NULL;
+ }
+ } else {
+ rb_erase(&cell->node, &prison->cells);
+ return true;
+ }
+ }
+
+ return false;
+}
+
+bool dm_cell_put_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell)
+{
+ bool r;
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ r = __put(prison, cell);
+ spin_unlock_irqrestore(&prison->lock, flags);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_put_v2);
+
+static int __lock(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned int lock_level,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell_result)
+{
+ struct dm_bio_prison_cell_v2 *cell;
+
+ if (__find_or_insert(prison, key, cell_prealloc, &cell)) {
+ if (cell->exclusive_lock)
+ return -EBUSY;
+
+ cell->exclusive_lock = true;
+ cell->exclusive_level = lock_level;
+ *cell_result = cell;
+
+ // FIXME: we don't yet know what level these shared locks
+ // were taken at, so have to quiesce them all.
+ return cell->shared_count > 0;
+
+ } else {
+ cell = cell_prealloc;
+ cell->shared_count = 0;
+ cell->exclusive_lock = true;
+ cell->exclusive_level = lock_level;
+ *cell_result = cell;
+ }
+
+ return 0;
+}
+
+int dm_cell_lock_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned int lock_level,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell_result)
+{
+ int r;
+
+ spin_lock_irq(&prison->lock);
+ r = __lock(prison, key, lock_level, cell_prealloc, cell_result);
+ spin_unlock_irq(&prison->lock);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_lock_v2);
+
+static void __quiesce(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct work_struct *continuation)
+{
+ if (!cell->shared_count)
+ queue_work(prison->wq, continuation);
+ else
+ cell->quiesce_continuation = continuation;
+}
+
+void dm_cell_quiesce_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct work_struct *continuation)
+{
+ spin_lock_irq(&prison->lock);
+ __quiesce(prison, cell, continuation);
+ spin_unlock_irq(&prison->lock);
+}
+EXPORT_SYMBOL_GPL(dm_cell_quiesce_v2);
+
+static int __promote(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ unsigned int new_lock_level)
+{
+ if (!cell->exclusive_lock)
+ return -EINVAL;
+
+ cell->exclusive_level = new_lock_level;
+ return cell->shared_count > 0;
+}
+
+int dm_cell_lock_promote_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ unsigned int new_lock_level)
+{
+ int r;
+
+ spin_lock_irq(&prison->lock);
+ r = __promote(prison, cell, new_lock_level);
+ spin_unlock_irq(&prison->lock);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_lock_promote_v2);
+
+static bool __unlock(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct bio_list *bios)
+{
+ BUG_ON(!cell->exclusive_lock);
+
+ bio_list_merge(bios, &cell->bios);
+ bio_list_init(&cell->bios);
+
+ if (cell->shared_count) {
+ cell->exclusive_lock = false;
+ return false;
+ }
+
+ rb_erase(&cell->node, &prison->cells);
+ return true;
+}
+
+bool dm_cell_unlock_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct bio_list *bios)
+{
+ bool r;
+
+ spin_lock_irq(&prison->lock);
+ r = __unlock(prison, cell, bios);
+ spin_unlock_irq(&prison->lock);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_unlock_v2);
+
+/*----------------------------------------------------------------*/
+
+int __init dm_bio_prison_init_v2(void)
+{
+ _cell_cache = KMEM_CACHE(dm_bio_prison_cell_v2, 0);
+ if (!_cell_cache)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void dm_bio_prison_exit_v2(void)
+{
+ kmem_cache_destroy(_cell_cache);
+ _cell_cache = NULL;
+}