From 5d1646d90e1f2cceb9f0828f4b28318cd0ec7744 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 27 Apr 2024 12:05:51 +0200 Subject: Adding upstream version 5.10.209. Signed-off-by: Daniel Baumann --- drivers/gpu/drm/ttm/Makefile | 12 + drivers/gpu/drm/ttm/ttm_agp_backend.c | 146 +++ drivers/gpu/drm/ttm/ttm_bo.c | 1642 ++++++++++++++++++++++++++++++ drivers/gpu/drm/ttm/ttm_bo_util.c | 656 ++++++++++++ drivers/gpu/drm/ttm/ttm_bo_vm.c | 611 +++++++++++ drivers/gpu/drm/ttm/ttm_execbuf_util.c | 172 ++++ drivers/gpu/drm/ttm/ttm_memory.c | 683 +++++++++++++ drivers/gpu/drm/ttm/ttm_module.c | 103 ++ drivers/gpu/drm/ttm/ttm_page_alloc.c | 1189 +++++++++++++++++++++ drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 1239 ++++++++++++++++++++++ drivers/gpu/drm/ttm/ttm_range_manager.c | 185 ++++ drivers/gpu/drm/ttm/ttm_resource.c | 146 +++ drivers/gpu/drm/ttm/ttm_tt.c | 468 +++++++++ 13 files changed, 7252 insertions(+) create mode 100644 drivers/gpu/drm/ttm/Makefile create mode 100644 drivers/gpu/drm/ttm/ttm_agp_backend.c create mode 100644 drivers/gpu/drm/ttm/ttm_bo.c create mode 100644 drivers/gpu/drm/ttm/ttm_bo_util.c create mode 100644 drivers/gpu/drm/ttm/ttm_bo_vm.c create mode 100644 drivers/gpu/drm/ttm/ttm_execbuf_util.c create mode 100644 drivers/gpu/drm/ttm/ttm_memory.c create mode 100644 drivers/gpu/drm/ttm/ttm_module.c create mode 100644 drivers/gpu/drm/ttm/ttm_page_alloc.c create mode 100644 drivers/gpu/drm/ttm/ttm_page_alloc_dma.c create mode 100644 drivers/gpu/drm/ttm/ttm_range_manager.c create mode 100644 drivers/gpu/drm/ttm/ttm_resource.c create mode 100644 drivers/gpu/drm/ttm/ttm_tt.c (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile new file mode 100644 index 000000000..90c0da88c --- /dev/null +++ b/drivers/gpu/drm/ttm/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the drm device driver. This driver provides support for the + +ttm-y := ttm_memory.o ttm_tt.o ttm_bo.o \ + ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ + ttm_execbuf_util.o ttm_page_alloc.o ttm_range_manager.o \ + ttm_resource.o +ttm-$(CONFIG_AGP) += ttm_agp_backend.o +ttm-$(CONFIG_DRM_TTM_DMA_PAGE_POOL) += ttm_page_alloc_dma.o + +obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c new file mode 100644 index 000000000..a98fd795b --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + * Keith Packard. + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ttm_agp_backend { + struct ttm_tt ttm; + struct agp_memory *mem; + struct agp_bridge_data *bridge; +}; + +int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_resource *bo_mem) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + struct page *dummy_read_page = ttm_bo_glob.dummy_read_page; + struct drm_mm_node *node = bo_mem->mm_node; + struct agp_memory *mem; + int ret, cached = (bo_mem->placement & TTM_PL_FLAG_CACHED); + unsigned i; + + if (agp_be->mem) + return 0; + + mem = agp_allocate_memory(agp_be->bridge, ttm->num_pages, AGP_USER_MEMORY); + if (unlikely(mem == NULL)) + return -ENOMEM; + + mem->page_count = 0; + for (i = 0; i < ttm->num_pages; i++) { + struct page *page = ttm->pages[i]; + + if (!page) + page = dummy_read_page; + + mem->pages[mem->page_count++] = page; + } + agp_be->mem = mem; + + mem->is_flushed = 1; + mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY; + + ret = agp_bind_memory(mem, node->start); + if (ret) + pr_err("AGP Bind memory failed\n"); + + return ret; +} +EXPORT_SYMBOL(ttm_agp_bind); + +void ttm_agp_unbind(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) { + if (agp_be->mem->is_bound) { + agp_unbind_memory(agp_be->mem); + return; + } + agp_free_memory(agp_be->mem); + agp_be->mem = NULL; + } +} +EXPORT_SYMBOL(ttm_agp_unbind); + +bool ttm_agp_is_bound(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (!ttm) + return false; + + return (agp_be->mem != NULL); +} +EXPORT_SYMBOL(ttm_agp_is_bound); + +void ttm_agp_destroy(struct ttm_tt *ttm) +{ + struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); + + if (agp_be->mem) + ttm_agp_unbind(ttm); + ttm_tt_fini(ttm); + kfree(agp_be); +} +EXPORT_SYMBOL(ttm_agp_destroy); + +struct ttm_tt *ttm_agp_tt_create(struct ttm_buffer_object *bo, + struct agp_bridge_data *bridge, + uint32_t page_flags) +{ + struct ttm_agp_backend *agp_be; + + agp_be = kmalloc(sizeof(*agp_be), GFP_KERNEL); + if (!agp_be) + return NULL; + + agp_be->mem = NULL; + agp_be->bridge = bridge; + + if (ttm_tt_init(&agp_be->ttm, bo, page_flags)) { + kfree(agp_be); + return NULL; + } + + return &agp_be->ttm; +} +EXPORT_SYMBOL(ttm_agp_tt_create); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c new file mode 100644 index 000000000..8fe3be20a --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -0,0 +1,1642 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void ttm_bo_global_kobj_release(struct kobject *kobj); + +/** + * ttm_global_mutex - protecting the global BO state + */ +DEFINE_MUTEX(ttm_global_mutex); +unsigned ttm_bo_glob_use_count; +struct ttm_bo_global ttm_bo_glob; +EXPORT_SYMBOL(ttm_bo_glob); + +static struct attribute ttm_bo_count = { + .name = "bo_count", + .mode = S_IRUGO +}; + +/* default destructor */ +static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) +{ + kfree(bo); +} + +static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, + struct ttm_placement *placement) +{ + struct drm_printer p = drm_debug_printer(TTM_PFX); + struct ttm_resource_manager *man; + int i, mem_type; + + drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n", + bo, bo->mem.num_pages, bo->mem.size >> 10, + bo->mem.size >> 20); + for (i = 0; i < placement->num_placement; i++) { + mem_type = placement->placement[i].mem_type; + drm_printf(&p, " placement[%d]=0x%08X (%d)\n", + i, placement->placement[i].flags, mem_type); + man = ttm_manager_type(bo->bdev, mem_type); + ttm_resource_manager_debug(man, &p); + } +} + +static ssize_t ttm_bo_global_show(struct kobject *kobj, + struct attribute *attr, + char *buffer) +{ + struct ttm_bo_global *glob = + container_of(kobj, struct ttm_bo_global, kobj); + + return snprintf(buffer, PAGE_SIZE, "%d\n", + atomic_read(&glob->bo_count)); +} + +static struct attribute *ttm_bo_global_attrs[] = { + &ttm_bo_count, + NULL +}; + +static const struct sysfs_ops ttm_bo_global_ops = { + .show = &ttm_bo_global_show +}; + +static struct kobj_type ttm_bo_glob_kobj_type = { + .release = &ttm_bo_global_kobj_release, + .sysfs_ops = &ttm_bo_global_ops, + .default_attrs = ttm_bo_global_attrs +}; + +static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo, + struct ttm_resource *mem) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource_manager *man; + + if (!list_empty(&bo->lru) || bo->pin_count) + return; + + if (mem->placement & TTM_PL_FLAG_NO_EVICT) + return; + + man = ttm_manager_type(bdev, mem->mem_type); + list_add_tail(&bo->lru, &man->lru[bo->priority]); + + if (man->use_tt && bo->ttm && + !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG | + TTM_PAGE_FLAG_SWAPPED))) { + list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]); + } +} + +static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + bool notify = false; + + if (!list_empty(&bo->swap)) { + list_del_init(&bo->swap); + notify = true; + } + if (!list_empty(&bo->lru)) { + list_del_init(&bo->lru); + notify = true; + } + + if (notify && bdev->driver->del_from_lru_notify) + bdev->driver->del_from_lru_notify(bo); +} + +static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, + struct ttm_buffer_object *bo) +{ + if (!pos->first) + pos->first = bo; + pos->last = bo; +} + +void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, + struct ttm_lru_bulk_move *bulk) +{ + dma_resv_assert_held(bo->base.resv); + + ttm_bo_del_from_lru(bo); + ttm_bo_add_mem_to_lru(bo, &bo->mem); + + if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT) && + !bo->pin_count) { + switch (bo->mem.mem_type) { + case TTM_PL_TT: + ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); + break; + + case TTM_PL_VRAM: + ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo); + break; + } + if (bo->ttm && !(bo->ttm->page_flags & + (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) + ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo); + } +} +EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); + +void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) +{ + unsigned i; + + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i]; + struct ttm_resource_manager *man; + + if (!pos->first) + continue; + + dma_resv_assert_held(pos->first->base.resv); + dma_resv_assert_held(pos->last->base.resv); + + man = ttm_manager_type(pos->first->bdev, TTM_PL_TT); + list_bulk_move_tail(&man->lru[i], &pos->first->lru, + &pos->last->lru); + } + + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i]; + struct ttm_resource_manager *man; + + if (!pos->first) + continue; + + dma_resv_assert_held(pos->first->base.resv); + dma_resv_assert_held(pos->last->base.resv); + + man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM); + list_bulk_move_tail(&man->lru[i], &pos->first->lru, + &pos->last->lru); + } + + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i]; + struct list_head *lru; + + if (!pos->first) + continue; + + dma_resv_assert_held(pos->first->base.resv); + dma_resv_assert_held(pos->last->base.resv); + + lru = &ttm_bo_glob.swap_lru[i]; + list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap); + } +} +EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail); + +static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, + struct ttm_resource *mem, bool evict, + struct ttm_operation_ctx *ctx) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type); + struct ttm_resource_manager *new_man = ttm_manager_type(bdev, mem->mem_type); + int ret; + + ttm_bo_unmap_virtual(bo); + + /* + * Create and bind a ttm if required. + */ + + if (new_man->use_tt) { + /* Zero init the new TTM structure if the old location should + * have used one as well. + */ + ret = ttm_tt_create(bo, old_man->use_tt); + if (ret) + goto out_err; + + ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); + if (ret) + goto out_err; + + if (mem->mem_type != TTM_PL_SYSTEM) { + ret = ttm_tt_populate(bdev, bo->ttm, ctx); + if (ret) + goto out_err; + + ret = ttm_bo_tt_bind(bo, mem); + if (ret) + goto out_err; + } + + if (bo->mem.mem_type == TTM_PL_SYSTEM) { + if (bdev->driver->move_notify) + bdev->driver->move_notify(bo, evict, mem); + bo->mem = *mem; + goto moved; + } + } + + if (bdev->driver->move_notify) + bdev->driver->move_notify(bo, evict, mem); + + if (old_man->use_tt && new_man->use_tt) + ret = ttm_bo_move_ttm(bo, ctx, mem); + else if (bdev->driver->move) + ret = bdev->driver->move(bo, evict, ctx, mem); + else + ret = ttm_bo_move_memcpy(bo, ctx, mem); + + if (ret) { + if (bdev->driver->move_notify) { + swap(*mem, bo->mem); + bdev->driver->move_notify(bo, false, mem); + swap(*mem, bo->mem); + } + + goto out_err; + } + +moved: + ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; + return 0; + +out_err: + new_man = ttm_manager_type(bdev, bo->mem.mem_type); + if (!new_man->use_tt) + ttm_bo_tt_destroy(bo); + + return ret; +} + +/** + * Call bo::reserved. + * Will release GPU memory type usage on destruction. + * This is the place to put in driver specific hooks to release + * driver private resources. + * Will release the bo::reserved lock. + */ + +static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) +{ + if (bo->bdev->driver->move_notify) + bo->bdev->driver->move_notify(bo, false, NULL); + + ttm_bo_tt_destroy(bo); + ttm_resource_free(bo, &bo->mem); +} + +static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) +{ + int r; + + if (bo->base.resv == &bo->base._resv) + return 0; + + BUG_ON(!dma_resv_trylock(&bo->base._resv)); + + r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); + dma_resv_unlock(&bo->base._resv); + if (r) + return r; + + if (bo->type != ttm_bo_type_sg) { + /* This works because the BO is about to be destroyed and nobody + * reference it any more. The only tricky case is the trylock on + * the resv object while holding the lru_lock. + */ + spin_lock(&ttm_bo_glob.lru_lock); + bo->base.resv = &bo->base._resv; + spin_unlock(&ttm_bo_glob.lru_lock); + } + + return r; +} + +static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) +{ + struct dma_resv *resv = &bo->base._resv; + struct dma_resv_list *fobj; + struct dma_fence *fence; + int i; + + rcu_read_lock(); + fobj = rcu_dereference(resv->fence); + fence = rcu_dereference(resv->fence_excl); + if (fence && !fence->ops->signaled) + dma_fence_enable_sw_signaling(fence); + + for (i = 0; fobj && i < fobj->shared_count; ++i) { + fence = rcu_dereference(fobj->shared[i]); + + if (!fence->ops->signaled) + dma_fence_enable_sw_signaling(fence); + } + rcu_read_unlock(); +} + +/** + * function ttm_bo_cleanup_refs + * If bo idle, remove from lru lists, and unref. + * If not idle, block if possible. + * + * Must be called with lru_lock and reservation held, this function + * will drop the lru lock and optionally the reservation lock before returning. + * + * @interruptible Any sleeps should occur interruptibly. + * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. + * @unlock_resv Unlock the reservation lock as well. + */ + +static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, + bool interruptible, bool no_wait_gpu, + bool unlock_resv) +{ + struct dma_resv *resv = &bo->base._resv; + int ret; + + if (dma_resv_test_signaled_rcu(resv, true)) + ret = 0; + else + ret = -EBUSY; + + if (ret && !no_wait_gpu) { + long lret; + + if (unlock_resv) + dma_resv_unlock(bo->base.resv); + spin_unlock(&ttm_bo_glob.lru_lock); + + lret = dma_resv_wait_timeout_rcu(resv, true, interruptible, + 30 * HZ); + + if (lret < 0) + return lret; + else if (lret == 0) + return -EBUSY; + + spin_lock(&ttm_bo_glob.lru_lock); + if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { + /* + * We raced, and lost, someone else holds the reservation now, + * and is probably busy in ttm_bo_cleanup_memtype_use. + * + * Even if it's not the case, because we finished waiting any + * delayed destruction would succeed, so just return success + * here. + */ + spin_unlock(&ttm_bo_glob.lru_lock); + return 0; + } + ret = 0; + } + + if (ret || unlikely(list_empty(&bo->ddestroy))) { + if (unlock_resv) + dma_resv_unlock(bo->base.resv); + spin_unlock(&ttm_bo_glob.lru_lock); + return ret; + } + + ttm_bo_del_from_lru(bo); + list_del_init(&bo->ddestroy); + spin_unlock(&ttm_bo_glob.lru_lock); + ttm_bo_cleanup_memtype_use(bo); + + if (unlock_resv) + dma_resv_unlock(bo->base.resv); + + ttm_bo_put(bo); + + return 0; +} + +/** + * Traverse the delayed list, and call ttm_bo_cleanup_refs on all + * encountered buffers. + */ +static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +{ + struct ttm_bo_global *glob = &ttm_bo_glob; + struct list_head removed; + bool empty; + + INIT_LIST_HEAD(&removed); + + spin_lock(&glob->lru_lock); + while (!list_empty(&bdev->ddestroy)) { + struct ttm_buffer_object *bo; + + bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, + ddestroy); + list_move_tail(&bo->ddestroy, &removed); + if (!ttm_bo_get_unless_zero(bo)) + continue; + + if (remove_all || bo->base.resv != &bo->base._resv) { + spin_unlock(&glob->lru_lock); + dma_resv_lock(bo->base.resv, NULL); + + spin_lock(&glob->lru_lock); + ttm_bo_cleanup_refs(bo, false, !remove_all, true); + + } else if (dma_resv_trylock(bo->base.resv)) { + ttm_bo_cleanup_refs(bo, false, !remove_all, true); + } else { + spin_unlock(&glob->lru_lock); + } + + ttm_bo_put(bo); + spin_lock(&glob->lru_lock); + } + list_splice_tail(&removed, &bdev->ddestroy); + empty = list_empty(&bdev->ddestroy); + spin_unlock(&glob->lru_lock); + + return empty; +} + +static void ttm_bo_delayed_workqueue(struct work_struct *work) +{ + struct ttm_bo_device *bdev = + container_of(work, struct ttm_bo_device, wq.work); + + if (!ttm_bo_delayed_delete(bdev, false)) + schedule_delayed_work(&bdev->wq, + ((HZ / 100) < 1) ? 1 : HZ / 100); +} + +static void ttm_bo_release(struct kref *kref) +{ + struct ttm_buffer_object *bo = + container_of(kref, struct ttm_buffer_object, kref); + struct ttm_bo_device *bdev = bo->bdev; + size_t acc_size = bo->acc_size; + int ret; + + if (!bo->deleted) { + ret = ttm_bo_individualize_resv(bo); + if (ret) { + /* Last resort, if we fail to allocate memory for the + * fences block for the BO to become idle + */ + dma_resv_wait_timeout_rcu(bo->base.resv, true, false, + 30 * HZ); + } + + if (bo->bdev->driver->release_notify) + bo->bdev->driver->release_notify(bo); + + drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); + ttm_mem_io_free(bdev, &bo->mem); + } + + if (!dma_resv_test_signaled_rcu(bo->base.resv, true) || + !dma_resv_trylock(bo->base.resv)) { + /* The BO is not idle, resurrect it for delayed destroy */ + ttm_bo_flush_all_fences(bo); + bo->deleted = true; + + spin_lock(&ttm_bo_glob.lru_lock); + + /* + * Make NO_EVICT bos immediately available to + * shrinkers, now that they are queued for + * destruction. + */ + if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT || bo->pin_count) { + bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; + bo->pin_count = 0; + ttm_bo_del_from_lru(bo); + ttm_bo_add_mem_to_lru(bo, &bo->mem); + } + + kref_init(&bo->kref); + list_add_tail(&bo->ddestroy, &bdev->ddestroy); + spin_unlock(&ttm_bo_glob.lru_lock); + + schedule_delayed_work(&bdev->wq, + ((HZ / 100) < 1) ? 1 : HZ / 100); + return; + } + + spin_lock(&ttm_bo_glob.lru_lock); + ttm_bo_del_from_lru(bo); + list_del(&bo->ddestroy); + spin_unlock(&ttm_bo_glob.lru_lock); + + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + + atomic_dec(&ttm_bo_glob.bo_count); + dma_fence_put(bo->moving); + if (!ttm_bo_uses_embedded_gem_object(bo)) + dma_resv_fini(&bo->base._resv); + bo->destroy(bo); + ttm_mem_global_free(&ttm_mem_glob, acc_size); +} + +void ttm_bo_put(struct ttm_buffer_object *bo) +{ + kref_put(&bo->kref, ttm_bo_release); +} +EXPORT_SYMBOL(ttm_bo_put); + +int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) +{ + return cancel_delayed_work_sync(&bdev->wq); +} +EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); + +void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) +{ + if (resched) + schedule_delayed_work(&bdev->wq, + ((HZ / 100) < 1) ? 1 : HZ / 100); +} +EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); + +static int ttm_bo_evict(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource evict_mem; + struct ttm_placement placement; + int ret = 0; + + dma_resv_assert_held(bo->base.resv); + + placement.num_placement = 0; + placement.num_busy_placement = 0; + bdev->driver->evict_flags(bo, &placement); + + if (!placement.num_placement && !placement.num_busy_placement) { + ttm_bo_wait(bo, false, false); + + ttm_bo_cleanup_memtype_use(bo); + return ttm_tt_create(bo, false); + } + + evict_mem = bo->mem; + evict_mem.mm_node = NULL; + evict_mem.bus.offset = 0; + evict_mem.bus.addr = NULL; + + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); + if (ret) { + if (ret != -ERESTARTSYS) { + pr_err("Failed to find memory space for buffer 0x%p eviction\n", + bo); + ttm_bo_mem_space_debug(bo, &placement); + } + goto out; + } + + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); + if (unlikely(ret)) { + if (ret != -ERESTARTSYS) + pr_err("Buffer eviction failed\n"); + ttm_resource_free(bo, &evict_mem); + } +out: + return ret; +} + +bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, + const struct ttm_place *place) +{ + /* Don't evict this BO if it's outside of the + * requested placement range + */ + if (place->fpfn >= (bo->mem.start + bo->mem.num_pages) || + (place->lpfn && place->lpfn <= bo->mem.start)) + return false; + + return true; +} +EXPORT_SYMBOL(ttm_bo_eviction_valuable); + +/** + * Check the target bo is allowable to be evicted or swapout, including cases: + * + * a. if share same reservation object with ctx->resv, have assumption + * reservation objects should already be locked, so not lock again and + * return true directly when either the opreation allow_reserved_eviction + * or the target bo already is in delayed free list; + * + * b. Otherwise, trylock it. + */ +static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, bool *locked, bool *busy) +{ + bool ret = false; + + if (bo->pin_count) { + *locked = false; + if (busy) + *busy = false; + return false; + } + + if (bo->base.resv == ctx->resv) { + dma_resv_assert_held(bo->base.resv); + if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT) + ret = true; + *locked = false; + if (busy) + *busy = false; + } else { + ret = dma_resv_trylock(bo->base.resv); + *locked = ret; + if (busy) + *busy = !ret; + } + + return ret; +} + +/** + * ttm_mem_evict_wait_busy - wait for a busy BO to become available + * + * @busy_bo: BO which couldn't be locked with trylock + * @ctx: operation context + * @ticket: acquire ticket + * + * Try to lock a busy buffer object to avoid failing eviction. + */ +static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, + struct ttm_operation_ctx *ctx, + struct ww_acquire_ctx *ticket) +{ + int r; + + if (!busy_bo || !ticket) + return -EBUSY; + + if (ctx->interruptible) + r = dma_resv_lock_interruptible(busy_bo->base.resv, + ticket); + else + r = dma_resv_lock(busy_bo->base.resv, ticket); + + /* + * TODO: It would be better to keep the BO locked until allocation is at + * least tried one more time, but that would mean a much larger rework + * of TTM. + */ + if (!r) + dma_resv_unlock(busy_bo->base.resv); + + return r == -EDEADLK ? -EBUSY : r; +} + +int ttm_mem_evict_first(struct ttm_bo_device *bdev, + struct ttm_resource_manager *man, + const struct ttm_place *place, + struct ttm_operation_ctx *ctx, + struct ww_acquire_ctx *ticket) +{ + struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; + bool locked = false; + unsigned i; + int ret; + + spin_lock(&ttm_bo_glob.lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + list_for_each_entry(bo, &man->lru[i], lru) { + bool busy; + + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, + &busy)) { + if (busy && !busy_bo && ticket != + dma_resv_locking_ctx(bo->base.resv)) + busy_bo = bo; + continue; + } + + if (place && !bdev->driver->eviction_valuable(bo, + place)) { + if (locked) + dma_resv_unlock(bo->base.resv); + continue; + } + if (!ttm_bo_get_unless_zero(bo)) { + if (locked) + dma_resv_unlock(bo->base.resv); + continue; + } + break; + } + + /* If the inner loop terminated early, we have our candidate */ + if (&bo->lru != &man->lru[i]) + break; + + bo = NULL; + } + + if (!bo) { + if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) + busy_bo = NULL; + spin_unlock(&ttm_bo_glob.lru_lock); + ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); + if (busy_bo) + ttm_bo_put(busy_bo); + return ret; + } + + if (bo->deleted) { + ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, + ctx->no_wait_gpu, locked); + ttm_bo_put(bo); + return ret; + } + + spin_unlock(&ttm_bo_glob.lru_lock); + + ret = ttm_bo_evict(bo, ctx); + if (locked) + ttm_bo_unreserve(bo); + else + ttm_bo_move_to_lru_tail_unlocked(bo); + + ttm_bo_put(bo); + return ret; +} + +/** + * Add the last move fence to the BO and reserve a new shared slot. + */ +static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, + struct ttm_resource_manager *man, + struct ttm_resource *mem, + bool no_wait_gpu) +{ + struct dma_fence *fence; + int ret; + + spin_lock(&man->move_lock); + fence = dma_fence_get(man->move); + spin_unlock(&man->move_lock); + + if (!fence) + return 0; + + if (no_wait_gpu) { + dma_fence_put(fence); + return -EBUSY; + } + + dma_resv_add_shared_fence(bo->base.resv, fence); + + ret = dma_resv_reserve_shared(bo->base.resv, 1); + if (unlikely(ret)) { + dma_fence_put(fence); + return ret; + } + + dma_fence_put(bo->moving); + bo->moving = fence; + return 0; +} + +/** + * Repeatedly evict memory from the LRU for @mem_type until we create enough + * space, or we've evicted everything and there isn't enough space. + */ +static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource *mem, + struct ttm_operation_ctx *ctx) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource_manager *man = ttm_manager_type(bdev, mem->mem_type); + struct ww_acquire_ctx *ticket; + int ret; + + ticket = dma_resv_locking_ctx(bo->base.resv); + do { + ret = ttm_resource_alloc(bo, place, mem); + if (likely(!ret)) + break; + if (unlikely(ret != -ENOSPC)) + return ret; + ret = ttm_mem_evict_first(bdev, man, place, ctx, + ticket); + if (unlikely(ret != 0)) + return ret; + } while (1); + + return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu); +} + +static uint32_t ttm_bo_select_caching(struct ttm_resource_manager *man, + uint32_t cur_placement, + uint32_t proposed_placement) +{ + uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; + uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; + + /** + * Keep current caching if possible. + */ + + if ((cur_placement & caching) != 0) + result |= (cur_placement & caching); + else if ((TTM_PL_FLAG_CACHED & caching) != 0) + result |= TTM_PL_FLAG_CACHED; + else if ((TTM_PL_FLAG_WC & caching) != 0) + result |= TTM_PL_FLAG_WC; + else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) + result |= TTM_PL_FLAG_UNCACHED; + + return result; +} + +/** + * ttm_bo_mem_placement - check if placement is compatible + * @bo: BO to find memory for + * @place: where to search + * @mem: the memory object to fill in + * @ctx: operation context + * + * Check if placement is compatible and fill in mem structure. + * Returns -EBUSY if placement won't work or negative error code. + * 0 when placement can be used. + */ +static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource *mem, + struct ttm_operation_ctx *ctx) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource_manager *man; + uint32_t cur_flags = 0; + + man = ttm_manager_type(bdev, place->mem_type); + if (!man || !ttm_resource_manager_used(man)) + return -EBUSY; + + cur_flags = ttm_bo_select_caching(man, bo->mem.placement, + place->flags); + cur_flags |= place->flags & ~TTM_PL_MASK_CACHING; + + mem->mem_type = place->mem_type; + mem->placement = cur_flags; + + spin_lock(&ttm_bo_glob.lru_lock); + ttm_bo_del_from_lru(bo); + ttm_bo_add_mem_to_lru(bo, mem); + spin_unlock(&ttm_bo_glob.lru_lock); + + return 0; +} + +/** + * Creates space for memory region @mem according to its type. + * + * This function first searches for free space in compatible memory types in + * the priority order defined by the driver. If free space isn't found, then + * ttm_bo_mem_force_space is attempted in priority order to evict and find + * space. + */ +int ttm_bo_mem_space(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_resource *mem, + struct ttm_operation_ctx *ctx) +{ + struct ttm_bo_device *bdev = bo->bdev; + bool type_found = false; + int i, ret; + + ret = dma_resv_reserve_shared(bo->base.resv, 1); + if (unlikely(ret)) + return ret; + + for (i = 0; i < placement->num_placement; ++i) { + const struct ttm_place *place = &placement->placement[i]; + struct ttm_resource_manager *man; + + ret = ttm_bo_mem_placement(bo, place, mem, ctx); + if (ret) + continue; + + type_found = true; + ret = ttm_resource_alloc(bo, place, mem); + if (ret == -ENOSPC) + continue; + if (unlikely(ret)) + goto error; + + man = ttm_manager_type(bdev, mem->mem_type); + ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu); + if (unlikely(ret)) { + ttm_resource_free(bo, mem); + if (ret == -EBUSY) + continue; + + goto error; + } + return 0; + } + + for (i = 0; i < placement->num_busy_placement; ++i) { + const struct ttm_place *place = &placement->busy_placement[i]; + + ret = ttm_bo_mem_placement(bo, place, mem, ctx); + if (ret) + continue; + + type_found = true; + ret = ttm_bo_mem_force_space(bo, place, mem, ctx); + if (likely(!ret)) + return 0; + + if (ret && ret != -EBUSY) + goto error; + } + + ret = -ENOMEM; + if (!type_found) { + pr_err(TTM_PFX "No compatible memory type found\n"); + ret = -EINVAL; + } + +error: + if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) { + ttm_bo_move_to_lru_tail_unlocked(bo); + } + + return ret; +} +EXPORT_SYMBOL(ttm_bo_mem_space); + +static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) +{ + int ret = 0; + struct ttm_resource mem; + + dma_resv_assert_held(bo->base.resv); + + mem.num_pages = bo->num_pages; + mem.size = mem.num_pages << PAGE_SHIFT; + mem.page_alignment = bo->mem.page_alignment; + mem.bus.offset = 0; + mem.bus.addr = NULL; + mem.mm_node = NULL; + + /* + * Determine where to move the buffer. + */ + ret = ttm_bo_mem_space(bo, placement, &mem, ctx); + if (ret) + goto out_unlock; + ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); +out_unlock: + if (ret) + ttm_resource_free(bo, &mem); + return ret; +} + +static bool ttm_bo_places_compat(const struct ttm_place *places, + unsigned num_placement, + struct ttm_resource *mem, + uint32_t *new_flags) +{ + unsigned i; + + for (i = 0; i < num_placement; i++) { + const struct ttm_place *heap = &places[i]; + + if ((mem->start < heap->fpfn || + (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) + continue; + + *new_flags = heap->flags; + if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && + (mem->mem_type == heap->mem_type) && + (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || + (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) + return true; + } + return false; +} + +bool ttm_bo_mem_compat(struct ttm_placement *placement, + struct ttm_resource *mem, + uint32_t *new_flags) +{ + if (ttm_bo_places_compat(placement->placement, placement->num_placement, + mem, new_flags)) + return true; + + if ((placement->busy_placement != placement->placement || + placement->num_busy_placement > placement->num_placement) && + ttm_bo_places_compat(placement->busy_placement, + placement->num_busy_placement, + mem, new_flags)) + return true; + + return false; +} +EXPORT_SYMBOL(ttm_bo_mem_compat); + +int ttm_bo_validate(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) +{ + int ret; + uint32_t new_flags; + + dma_resv_assert_held(bo->base.resv); + + /* + * Remove the backing store if no placement is given. + */ + if (!placement->num_placement && !placement->num_busy_placement) { + ret = ttm_bo_pipeline_gutting(bo); + if (ret) + return ret; + + return ttm_tt_create(bo, false); + } + + /* + * Check whether we need to move buffer. + */ + if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { + ret = ttm_bo_move_buffer(bo, placement, ctx); + if (ret) + return ret; + } else { + bo->mem.placement &= TTM_PL_MASK_CACHING; + bo->mem.placement |= new_flags & ~TTM_PL_MASK_CACHING; + } + /* + * We might need to add a TTM. + */ + if (bo->mem.mem_type == TTM_PL_SYSTEM) { + ret = ttm_tt_create(bo, true); + if (ret) + return ret; + } + return 0; +} +EXPORT_SYMBOL(ttm_bo_validate); + +int ttm_bo_init_reserved(struct ttm_bo_device *bdev, + struct ttm_buffer_object *bo, + unsigned long size, + enum ttm_bo_type type, + struct ttm_placement *placement, + uint32_t page_alignment, + struct ttm_operation_ctx *ctx, + size_t acc_size, + struct sg_table *sg, + struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) +{ + struct ttm_mem_global *mem_glob = &ttm_mem_glob; + int ret = 0; + unsigned long num_pages; + bool locked; + + ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); + if (ret) { + pr_err("Out of kernel memory\n"); + if (destroy) + (*destroy)(bo); + else + kfree(bo); + return -ENOMEM; + } + + num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (num_pages == 0) { + pr_err("Illegal buffer object size\n"); + if (destroy) + (*destroy)(bo); + else + kfree(bo); + ttm_mem_global_free(mem_glob, acc_size); + return -EINVAL; + } + bo->destroy = destroy ? destroy : ttm_bo_default_destroy; + + kref_init(&bo->kref); + INIT_LIST_HEAD(&bo->lru); + INIT_LIST_HEAD(&bo->ddestroy); + INIT_LIST_HEAD(&bo->swap); + bo->bdev = bdev; + bo->type = type; + bo->num_pages = num_pages; + bo->mem.size = num_pages << PAGE_SHIFT; + bo->mem.mem_type = TTM_PL_SYSTEM; + bo->mem.num_pages = bo->num_pages; + bo->mem.mm_node = NULL; + bo->mem.page_alignment = page_alignment; + bo->mem.bus.offset = 0; + bo->mem.bus.addr = NULL; + bo->moving = NULL; + bo->mem.placement = TTM_PL_FLAG_CACHED; + bo->acc_size = acc_size; + bo->pin_count = 0; + bo->sg = sg; + if (resv) { + bo->base.resv = resv; + dma_resv_assert_held(bo->base.resv); + } else { + bo->base.resv = &bo->base._resv; + } + if (!ttm_bo_uses_embedded_gem_object(bo)) { + /* + * bo.gem is not initialized, so we have to setup the + * struct elements we want use regardless. + */ + dma_resv_init(&bo->base._resv); + drm_vma_node_reset(&bo->base.vma_node); + } + atomic_inc(&ttm_bo_glob.bo_count); + + /* + * For ttm_bo_type_device buffers, allocate + * address space from the device. + */ + if (bo->type == ttm_bo_type_device || + bo->type == ttm_bo_type_sg) + ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, + bo->mem.num_pages); + + /* passed reservation objects should already be locked, + * since otherwise lockdep will be angered in radeon. + */ + if (!resv) { + locked = dma_resv_trylock(bo->base.resv); + WARN_ON(!locked); + } + + if (likely(!ret)) + ret = ttm_bo_validate(bo, placement, ctx); + + if (unlikely(ret)) { + if (!resv) + ttm_bo_unreserve(bo); + + ttm_bo_put(bo); + return ret; + } + + ttm_bo_move_to_lru_tail_unlocked(bo); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_init_reserved); + +int ttm_bo_init(struct ttm_bo_device *bdev, + struct ttm_buffer_object *bo, + unsigned long size, + enum ttm_bo_type type, + struct ttm_placement *placement, + uint32_t page_alignment, + bool interruptible, + size_t acc_size, + struct sg_table *sg, + struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) +{ + struct ttm_operation_ctx ctx = { interruptible, false }; + int ret; + + ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, + page_alignment, &ctx, acc_size, + sg, resv, destroy); + if (ret) + return ret; + + if (!resv) + ttm_bo_unreserve(bo); + + return 0; +} +EXPORT_SYMBOL(ttm_bo_init); + +static size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, + unsigned long bo_size, + unsigned struct_size) +{ + unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; + size_t size = 0; + + size += ttm_round_pot(struct_size); + size += ttm_round_pot(npages * sizeof(void *)); + size += ttm_round_pot(sizeof(struct ttm_tt)); + return size; +} + +size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, + unsigned long bo_size, + unsigned struct_size) +{ + unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; + size_t size = 0; + + size += ttm_round_pot(struct_size); + size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); + size += ttm_round_pot(sizeof(struct ttm_dma_tt)); + return size; +} +EXPORT_SYMBOL(ttm_bo_dma_acc_size); + +int ttm_bo_create(struct ttm_bo_device *bdev, + unsigned long size, + enum ttm_bo_type type, + struct ttm_placement *placement, + uint32_t page_alignment, + bool interruptible, + struct ttm_buffer_object **p_bo) +{ + struct ttm_buffer_object *bo; + size_t acc_size; + int ret; + + bo = kzalloc(sizeof(*bo), GFP_KERNEL); + if (unlikely(bo == NULL)) + return -ENOMEM; + + acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); + ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, + interruptible, acc_size, + NULL, NULL, NULL); + if (likely(ret == 0)) + *p_bo = bo; + + return ret; +} +EXPORT_SYMBOL(ttm_bo_create); + +int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) +{ + struct ttm_resource_manager *man = ttm_manager_type(bdev, mem_type); + + if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { + pr_err("Illegal memory manager memory type %u\n", mem_type); + return -EINVAL; + } + + if (!man) { + pr_err("Memory type %u has not been initialized\n", mem_type); + return 0; + } + + return ttm_resource_manager_force_list_clean(bdev, man); +} +EXPORT_SYMBOL(ttm_bo_evict_mm); + +static void ttm_bo_global_kobj_release(struct kobject *kobj) +{ + struct ttm_bo_global *glob = + container_of(kobj, struct ttm_bo_global, kobj); + + __free_page(glob->dummy_read_page); +} + +static void ttm_bo_global_release(void) +{ + struct ttm_bo_global *glob = &ttm_bo_glob; + + mutex_lock(&ttm_global_mutex); + if (--ttm_bo_glob_use_count > 0) + goto out; + + kobject_del(&glob->kobj); + kobject_put(&glob->kobj); + ttm_mem_global_release(&ttm_mem_glob); + memset(glob, 0, sizeof(*glob)); +out: + mutex_unlock(&ttm_global_mutex); +} + +static int ttm_bo_global_init(void) +{ + struct ttm_bo_global *glob = &ttm_bo_glob; + int ret = 0; + unsigned i; + + mutex_lock(&ttm_global_mutex); + if (++ttm_bo_glob_use_count > 1) + goto out; + + ret = ttm_mem_global_init(&ttm_mem_glob); + if (ret) + goto out; + + spin_lock_init(&glob->lru_lock); + glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); + + if (unlikely(glob->dummy_read_page == NULL)) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + INIT_LIST_HEAD(&glob->swap_lru[i]); + INIT_LIST_HEAD(&glob->device_list); + atomic_set(&glob->bo_count, 0); + + ret = kobject_init_and_add( + &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); + if (unlikely(ret != 0)) + kobject_put(&glob->kobj); +out: + mutex_unlock(&ttm_global_mutex); + return ret; +} + +int ttm_bo_device_release(struct ttm_bo_device *bdev) +{ + struct ttm_bo_global *glob = &ttm_bo_glob; + int ret = 0; + unsigned i; + struct ttm_resource_manager *man; + + man = ttm_manager_type(bdev, TTM_PL_SYSTEM); + ttm_resource_manager_set_used(man, false); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, NULL); + + mutex_lock(&ttm_global_mutex); + list_del(&bdev->device_list); + mutex_unlock(&ttm_global_mutex); + + cancel_delayed_work_sync(&bdev->wq); + + if (ttm_bo_delayed_delete(bdev, true)) + pr_debug("Delayed destroy list was clean\n"); + + spin_lock(&glob->lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + if (list_empty(&man->lru[0])) + pr_debug("Swap list %d was clean\n", i); + spin_unlock(&glob->lru_lock); + + if (!ret) + ttm_bo_global_release(); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_device_release); + +static void ttm_bo_init_sysman(struct ttm_bo_device *bdev) +{ + struct ttm_resource_manager *man = &bdev->sysman; + + /* + * Initialize the system memory buffer type. + * Other types need to be driver / IOCTL initialized. + */ + man->use_tt = true; + + ttm_resource_manager_init(man, 0); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man); + ttm_resource_manager_set_used(man, true); +} + +int ttm_bo_device_init(struct ttm_bo_device *bdev, + struct ttm_bo_driver *driver, + struct address_space *mapping, + struct drm_vma_offset_manager *vma_manager, + bool need_dma32) +{ + struct ttm_bo_global *glob = &ttm_bo_glob; + int ret; + + if (WARN_ON(vma_manager == NULL)) + return -EINVAL; + + ret = ttm_bo_global_init(); + if (ret) + return ret; + + bdev->driver = driver; + + ttm_bo_init_sysman(bdev); + + bdev->vma_manager = vma_manager; + INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); + INIT_LIST_HEAD(&bdev->ddestroy); + bdev->dev_mapping = mapping; + bdev->need_dma32 = need_dma32; + mutex_lock(&ttm_global_mutex); + list_add_tail(&bdev->device_list, &glob->device_list); + mutex_unlock(&ttm_global_mutex); + + return 0; +} +EXPORT_SYMBOL(ttm_bo_device_init); + +/* + * buffer object vm functions. + */ + +void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) +{ + struct ttm_bo_device *bdev = bo->bdev; + + drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); + ttm_mem_io_free(bdev, &bo->mem); +} +EXPORT_SYMBOL(ttm_bo_unmap_virtual); + +int ttm_bo_wait(struct ttm_buffer_object *bo, + bool interruptible, bool no_wait) +{ + long timeout = 15 * HZ; + + if (no_wait) { + if (dma_resv_test_signaled_rcu(bo->base.resv, true)) + return 0; + else + return -EBUSY; + } + + timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true, + interruptible, timeout); + if (timeout < 0) + return timeout; + + if (timeout == 0) + return -EBUSY; + + dma_resv_add_excl_fence(bo->base.resv, NULL); + return 0; +} +EXPORT_SYMBOL(ttm_bo_wait); + +/** + * A buffer object shrink method that tries to swap out the first + * buffer object on the bo_global::swap_lru list. + */ +int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) +{ + struct ttm_buffer_object *bo; + int ret = -EBUSY; + bool locked; + unsigned i; + + spin_lock(&glob->lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + list_for_each_entry(bo, &glob->swap_lru[i], swap) { + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, + NULL)) + continue; + + if (!ttm_bo_get_unless_zero(bo)) { + if (locked) + dma_resv_unlock(bo->base.resv); + continue; + } + + ret = 0; + break; + } + if (!ret) + break; + } + + if (ret) { + spin_unlock(&glob->lru_lock); + return ret; + } + + if (bo->deleted) { + ret = ttm_bo_cleanup_refs(bo, false, false, locked); + ttm_bo_put(bo); + return ret; + } + + ttm_bo_del_from_lru(bo); + spin_unlock(&glob->lru_lock); + + /** + * Move to system cached + */ + + if (bo->mem.mem_type != TTM_PL_SYSTEM || + bo->ttm->caching_state != tt_cached) { + struct ttm_operation_ctx ctx = { false, false }; + struct ttm_resource evict_mem; + + evict_mem = bo->mem; + evict_mem.mm_node = NULL; + evict_mem.placement = TTM_PL_FLAG_CACHED; + evict_mem.mem_type = TTM_PL_SYSTEM; + + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); + if (unlikely(ret != 0)) + goto out; + } + + /** + * Make sure BO is idle. + */ + + ret = ttm_bo_wait(bo, false, false); + if (unlikely(ret != 0)) + goto out; + + ttm_bo_unmap_virtual(bo); + + /** + * Swap out. Buffer will be swapped in again as soon as + * anyone tries to access a ttm page. + */ + + if (bo->bdev->driver->swap_notify) + bo->bdev->driver->swap_notify(bo); + + ret = ttm_tt_swapout(bo->bdev, bo->ttm, bo->persistent_swap_storage); +out: + + /** + * + * Unreserve without putting on LRU to avoid swapping out an + * already swapped buffer. + */ + if (locked) + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); + return ret; +} +EXPORT_SYMBOL(ttm_bo_swapout); + +void ttm_bo_swapout_all(void) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + + while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0); +} +EXPORT_SYMBOL(ttm_bo_swapout_all); + +void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) +{ + if (bo->ttm == NULL) + return; + + ttm_tt_destroy(bo->bdev, bo->ttm); + bo->ttm = NULL; +} + +int ttm_bo_tt_bind(struct ttm_buffer_object *bo, struct ttm_resource *mem) +{ + return bo->bdev->driver->ttm_tt_bind(bo->bdev, bo->ttm, mem); +} + +void ttm_bo_tt_unbind(struct ttm_buffer_object *bo) +{ + bo->bdev->driver->ttm_tt_unbind(bo->bdev, bo->ttm); +} diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c new file mode 100644 index 000000000..1968df974 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -0,0 +1,656 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ttm_transfer_obj { + struct ttm_buffer_object base; + struct ttm_buffer_object *bo; +}; + +void ttm_bo_free_old_node(struct ttm_buffer_object *bo) +{ + ttm_resource_free(bo, &bo->mem); +} + +int ttm_bo_move_ttm(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + struct ttm_resource *new_mem) +{ + struct ttm_tt *ttm = bo->ttm; + struct ttm_resource *old_mem = &bo->mem; + int ret; + + if (old_mem->mem_type != TTM_PL_SYSTEM) { + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); + + if (unlikely(ret != 0)) { + if (ret != -ERESTARTSYS) + pr_err("Failed to expire sync object before unbinding TTM\n"); + return ret; + } + + ttm_bo_tt_unbind(bo); + ttm_bo_free_old_node(bo); + old_mem->mem_type = TTM_PL_SYSTEM; + } + + ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); + if (unlikely(ret != 0)) + return ret; + + if (new_mem->mem_type != TTM_PL_SYSTEM) { + + ret = ttm_tt_populate(bo->bdev, ttm, ctx); + if (unlikely(ret != 0)) + return ret; + + ret = ttm_bo_tt_bind(bo, new_mem); + if (unlikely(ret != 0)) + return ret; + } + + ttm_bo_assign_mem(bo, new_mem); + return 0; +} +EXPORT_SYMBOL(ttm_bo_move_ttm); + +int ttm_mem_io_reserve(struct ttm_bo_device *bdev, + struct ttm_resource *mem) +{ + if (mem->bus.offset || mem->bus.addr) + return 0; + + mem->bus.is_iomem = false; + if (!bdev->driver->io_mem_reserve) + return 0; + + return bdev->driver->io_mem_reserve(bdev, mem); +} + +void ttm_mem_io_free(struct ttm_bo_device *bdev, + struct ttm_resource *mem) +{ + if (!mem->bus.offset && !mem->bus.addr) + return; + + if (bdev->driver->io_mem_free) + bdev->driver->io_mem_free(bdev, mem); + + mem->bus.offset = 0; + mem->bus.addr = NULL; +} + +static int ttm_resource_ioremap(struct ttm_bo_device *bdev, + struct ttm_resource *mem, + void **virtual) +{ + int ret; + void *addr; + + *virtual = NULL; + ret = ttm_mem_io_reserve(bdev, mem); + if (ret || !mem->bus.is_iomem) + return ret; + + if (mem->bus.addr) { + addr = mem->bus.addr; + } else { + size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; + + if (mem->placement & TTM_PL_FLAG_WC) + addr = ioremap_wc(mem->bus.offset, bus_size); + else + addr = ioremap(mem->bus.offset, bus_size); + if (!addr) { + ttm_mem_io_free(bdev, mem); + return -ENOMEM; + } + } + *virtual = addr; + return 0; +} + +static void ttm_resource_iounmap(struct ttm_bo_device *bdev, + struct ttm_resource *mem, + void *virtual) +{ + if (virtual && mem->bus.addr == NULL) + iounmap(virtual); + ttm_mem_io_free(bdev, mem); +} + +static int ttm_copy_io_page(void *dst, void *src, unsigned long page) +{ + uint32_t *dstP = + (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); + uint32_t *srcP = + (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); + + int i; + for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) + iowrite32(ioread32(srcP++), dstP++); + return 0; +} + +static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, + unsigned long page, + pgprot_t prot) +{ + struct page *d = ttm->pages[page]; + void *dst; + + if (!d) + return -ENOMEM; + + src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); + dst = kmap_atomic_prot(d, prot); + if (!dst) + return -ENOMEM; + + memcpy_fromio(dst, src, PAGE_SIZE); + + kunmap_atomic(dst); + + return 0; +} + +static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, + unsigned long page, + pgprot_t prot) +{ + struct page *s = ttm->pages[page]; + void *src; + + if (!s) + return -ENOMEM; + + dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); + src = kmap_atomic_prot(s, prot); + if (!src) + return -ENOMEM; + + memcpy_toio(dst, src, PAGE_SIZE); + + kunmap_atomic(src); + + return 0; +} + +int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + struct ttm_resource *new_mem) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); + struct ttm_tt *ttm = bo->ttm; + struct ttm_resource *old_mem = &bo->mem; + struct ttm_resource old_copy = *old_mem; + void *old_iomap; + void *new_iomap; + int ret; + unsigned long i; + unsigned long page; + unsigned long add = 0; + int dir; + + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); + if (ret) + return ret; + + ret = ttm_resource_ioremap(bdev, old_mem, &old_iomap); + if (ret) + return ret; + ret = ttm_resource_ioremap(bdev, new_mem, &new_iomap); + if (ret) + goto out; + + /* + * Single TTM move. NOP. + */ + if (old_iomap == NULL && new_iomap == NULL) + goto out2; + + /* + * Don't move nonexistent data. Clear destination instead. + */ + if (old_iomap == NULL && + (ttm == NULL || (!ttm_tt_is_populated(ttm) && + !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { + memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); + goto out2; + } + + /* + * TTM might be null for moves within the same region. + */ + if (ttm) { + ret = ttm_tt_populate(bdev, ttm, ctx); + if (ret) + goto out1; + } + + add = 0; + dir = 1; + + if ((old_mem->mem_type == new_mem->mem_type) && + (new_mem->start < old_mem->start + old_mem->size)) { + dir = -1; + add = new_mem->num_pages - 1; + } + + for (i = 0; i < new_mem->num_pages; ++i) { + page = i * dir + add; + if (old_iomap == NULL) { + pgprot_t prot = ttm_io_prot(old_mem->placement, + PAGE_KERNEL); + ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, + prot); + } else if (new_iomap == NULL) { + pgprot_t prot = ttm_io_prot(new_mem->placement, + PAGE_KERNEL); + ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, + prot); + } else { + ret = ttm_copy_io_page(new_iomap, old_iomap, page); + } + if (ret) + goto out1; + } + mb(); +out2: + old_copy = *old_mem; + + ttm_bo_assign_mem(bo, new_mem); + + if (!man->use_tt) + ttm_bo_tt_destroy(bo); + +out1: + ttm_resource_iounmap(bdev, old_mem, new_iomap); +out: + ttm_resource_iounmap(bdev, &old_copy, old_iomap); + + /* + * On error, keep the mm node! + */ + if (!ret) + ttm_resource_free(bo, &old_copy); + return ret; +} +EXPORT_SYMBOL(ttm_bo_move_memcpy); + +static void ttm_transfered_destroy(struct ttm_buffer_object *bo) +{ + struct ttm_transfer_obj *fbo; + + fbo = container_of(bo, struct ttm_transfer_obj, base); + ttm_bo_put(fbo->bo); + kfree(fbo); +} + +/** + * ttm_buffer_object_transfer + * + * @bo: A pointer to a struct ttm_buffer_object. + * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, + * holding the data of @bo with the old placement. + * + * This is a utility function that may be called after an accelerated move + * has been scheduled. A new buffer object is created as a placeholder for + * the old data while it's being copied. When that buffer object is idle, + * it can be destroyed, releasing the space of the old placement. + * Returns: + * !0: Failure. + */ + +static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, + struct ttm_buffer_object **new_obj) +{ + struct ttm_transfer_obj *fbo; + int ret; + + fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); + if (!fbo) + return -ENOMEM; + + fbo->base = *bo; + + ttm_bo_get(bo); + fbo->bo = bo; + + /** + * Fix up members that we shouldn't copy directly: + * TODO: Explicit member copy would probably be better here. + */ + + atomic_inc(&ttm_bo_glob.bo_count); + INIT_LIST_HEAD(&fbo->base.ddestroy); + INIT_LIST_HEAD(&fbo->base.lru); + INIT_LIST_HEAD(&fbo->base.swap); + fbo->base.moving = NULL; + drm_vma_node_reset(&fbo->base.base.vma_node); + + kref_init(&fbo->base.kref); + fbo->base.destroy = &ttm_transfered_destroy; + fbo->base.acc_size = 0; + fbo->base.pin_count = 1; + if (bo->type != ttm_bo_type_sg) + fbo->base.base.resv = &fbo->base.base._resv; + + dma_resv_init(&fbo->base.base._resv); + fbo->base.base.dev = NULL; + ret = dma_resv_trylock(&fbo->base.base._resv); + WARN_ON(!ret); + + *new_obj = &fbo->base; + return 0; +} + +pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) +{ + /* Cached mappings need no adjustment */ + if (caching_flags & TTM_PL_FLAG_CACHED) + return tmp; + +#if defined(__i386__) || defined(__x86_64__) + if (caching_flags & TTM_PL_FLAG_WC) + tmp = pgprot_writecombine(tmp); + else if (boot_cpu_data.x86 > 3) + tmp = pgprot_noncached(tmp); +#endif +#if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ + defined(__powerpc__) || defined(__mips__) + if (caching_flags & TTM_PL_FLAG_WC) + tmp = pgprot_writecombine(tmp); + else + tmp = pgprot_noncached(tmp); +#endif +#if defined(__sparc__) + tmp = pgprot_noncached(tmp); +#endif + return tmp; +} +EXPORT_SYMBOL(ttm_io_prot); + +static int ttm_bo_ioremap(struct ttm_buffer_object *bo, + unsigned long offset, + unsigned long size, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_resource *mem = &bo->mem; + + if (bo->mem.bus.addr) { + map->bo_kmap_type = ttm_bo_map_premapped; + map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); + } else { + map->bo_kmap_type = ttm_bo_map_iomap; + if (mem->placement & TTM_PL_FLAG_WC) + map->virtual = ioremap_wc(bo->mem.bus.offset + offset, + size); + else + map->virtual = ioremap(bo->mem.bus.offset + offset, + size); + } + return (!map->virtual) ? -ENOMEM : 0; +} + +static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, + unsigned long start_page, + unsigned long num_pages, + struct ttm_bo_kmap_obj *map) +{ + struct ttm_resource *mem = &bo->mem; + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + struct ttm_tt *ttm = bo->ttm; + pgprot_t prot; + int ret; + + BUG_ON(!ttm); + + ret = ttm_tt_populate(bo->bdev, ttm, &ctx); + if (ret) + return ret; + + if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { + /* + * We're mapping a single page, and the desired + * page protection is consistent with the bo. + */ + + map->bo_kmap_type = ttm_bo_map_kmap; + map->page = ttm->pages[start_page]; + map->virtual = kmap(map->page); + } else { + /* + * We need to use vmap to get the desired page protection + * or to make the buffer object look contiguous. + */ + prot = ttm_io_prot(mem->placement, PAGE_KERNEL); + map->bo_kmap_type = ttm_bo_map_vmap; + map->virtual = vmap(ttm->pages + start_page, num_pages, + 0, prot); + } + return (!map->virtual) ? -ENOMEM : 0; +} + +int ttm_bo_kmap(struct ttm_buffer_object *bo, + unsigned long start_page, unsigned long num_pages, + struct ttm_bo_kmap_obj *map) +{ + unsigned long offset, size; + int ret; + + map->virtual = NULL; + map->bo = bo; + if (num_pages > bo->num_pages) + return -EINVAL; + if (start_page > bo->num_pages) + return -EINVAL; + + ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); + if (ret) + return ret; + if (!bo->mem.bus.is_iomem) { + return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); + } else { + offset = start_page << PAGE_SHIFT; + size = num_pages << PAGE_SHIFT; + return ttm_bo_ioremap(bo, offset, size, map); + } +} +EXPORT_SYMBOL(ttm_bo_kmap); + +void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) +{ + if (!map->virtual) + return; + switch (map->bo_kmap_type) { + case ttm_bo_map_iomap: + iounmap(map->virtual); + break; + case ttm_bo_map_vmap: + vunmap(map->virtual); + break; + case ttm_bo_map_kmap: + kunmap(map->page); + break; + case ttm_bo_map_premapped: + break; + default: + BUG(); + } + ttm_mem_io_free(map->bo->bdev, &map->bo->mem); + map->virtual = NULL; + map->page = NULL; +} +EXPORT_SYMBOL(ttm_bo_kunmap); + +static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, + bool dst_use_tt) +{ + int ret; + ret = ttm_bo_wait(bo, false, false); + if (ret) + return ret; + + if (!dst_use_tt) + ttm_bo_tt_destroy(bo); + ttm_bo_free_old_node(bo); + return 0; +} + +static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, + struct dma_fence *fence, + bool dst_use_tt) +{ + struct ttm_buffer_object *ghost_obj; + int ret; + + /** + * This should help pipeline ordinary buffer moves. + * + * Hang old buffer memory on a new buffer object, + * and leave it to be released when the GPU + * operation has completed. + */ + + dma_fence_put(bo->moving); + bo->moving = dma_fence_get(fence); + + ret = ttm_buffer_object_transfer(bo, &ghost_obj); + if (ret) + return ret; + + dma_resv_add_excl_fence(&ghost_obj->base._resv, fence); + + /** + * If we're not moving to fixed memory, the TTM object + * needs to stay alive. Otherwhise hang it on the ghost + * bo to be unbound and destroyed. + */ + + if (dst_use_tt) + ghost_obj->ttm = NULL; + else + bo->ttm = NULL; + + dma_resv_unlock(&ghost_obj->base._resv); + ttm_bo_put(ghost_obj); + return 0; +} + +static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, + struct dma_fence *fence) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); + + /** + * BO doesn't have a TTM we need to bind/unbind. Just remember + * this eviction and free up the allocation + */ + spin_lock(&from->move_lock); + if (!from->move || dma_fence_is_later(fence, from->move)) { + dma_fence_put(from->move); + from->move = dma_fence_get(fence); + } + spin_unlock(&from->move_lock); + + ttm_bo_free_old_node(bo); + + dma_fence_put(bo->moving); + bo->moving = dma_fence_get(fence); +} + +int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, + struct dma_fence *fence, + bool evict, + bool pipeline, + struct ttm_resource *new_mem) +{ + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); + struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); + int ret = 0; + + dma_resv_add_excl_fence(bo->base.resv, fence); + if (!evict) + ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt); + else if (!from->use_tt && pipeline) + ttm_bo_move_pipeline_evict(bo, fence); + else + ret = ttm_bo_wait_free_node(bo, man->use_tt); + + if (ret) + return ret; + + ttm_bo_assign_mem(bo, new_mem); + + return 0; +} +EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); + +int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) +{ + struct ttm_buffer_object *ghost; + int ret; + + ret = ttm_buffer_object_transfer(bo, &ghost); + if (ret) + return ret; + + ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); + /* Last resort, wait for the BO to be idle when we are OOM */ + if (ret) + ttm_bo_wait(bo, false, false); + + memset(&bo->mem, 0, sizeof(bo->mem)); + bo->mem.mem_type = TTM_PL_SYSTEM; + bo->ttm = NULL; + + dma_resv_unlock(&ghost->base._resv); + ttm_bo_put(ghost); + + return 0; +} diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c new file mode 100644 index 000000000..0b1daf442 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -0,0 +1,611 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, + struct vm_fault *vmf) +{ + vm_fault_t ret = 0; + int err = 0; + + if (likely(!bo->moving)) + goto out_unlock; + + /* + * Quick non-stalling check for idle. + */ + if (dma_fence_is_signaled(bo->moving)) + goto out_clear; + + /* + * If possible, avoid waiting for GPU with mmap_lock + * held. We only do this if the fault allows retry and this + * is the first attempt. + */ + if (fault_flag_allow_retry_first(vmf->flags)) { + ret = VM_FAULT_RETRY; + if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) + goto out_unlock; + + ttm_bo_get(bo); + mmap_read_unlock(vmf->vma->vm_mm); + (void) dma_fence_wait(bo->moving, true); + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); + goto out_unlock; + } + + /* + * Ordinary wait. + */ + err = dma_fence_wait(bo->moving, true); + if (unlikely(err != 0)) { + ret = (err != -ERESTARTSYS) ? VM_FAULT_SIGBUS : + VM_FAULT_NOPAGE; + goto out_unlock; + } + +out_clear: + dma_fence_put(bo->moving); + bo->moving = NULL; + +out_unlock: + return ret; +} + +static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct ttm_bo_device *bdev = bo->bdev; + + if (bdev->driver->io_mem_pfn) + return bdev->driver->io_mem_pfn(bo, page_offset); + + return (bo->mem.bus.offset >> PAGE_SHIFT) + page_offset; +} + +/** + * ttm_bo_vm_reserve - Reserve a buffer object in a retryable vm callback + * @bo: The buffer object + * @vmf: The fault structure handed to the callback + * + * vm callbacks like fault() and *_mkwrite() allow for the mm_sem to be dropped + * during long waits, and after the wait the callback will be restarted. This + * is to allow other threads using the same virtual memory space concurrent + * access to map(), unmap() completely unrelated buffer objects. TTM buffer + * object reservations sometimes wait for GPU and should therefore be + * considered long waits. This function reserves the buffer object interruptibly + * taking this into account. Starvation is avoided by the vm system not + * allowing too many repeated restarts. + * This function is intended to be used in customized fault() and _mkwrite() + * handlers. + * + * Return: + * 0 on success and the bo was reserved. + * VM_FAULT_RETRY if blocking wait. + * VM_FAULT_NOPAGE if blocking wait and retrying was not allowed. + */ +vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, + struct vm_fault *vmf) +{ + /* + * Work around locking order reversal in fault / nopfn + * between mmap_lock and bo_reserve: Perform a trylock operation + * for reserve, and if it fails, retry the fault after waiting + * for the buffer to become unreserved. + */ + if (unlikely(!dma_resv_trylock(bo->base.resv))) { + /* + * If the fault allows retry and this is the first + * fault attempt, we try to release the mmap_lock + * before waiting + */ + if (fault_flag_allow_retry_first(vmf->flags)) { + if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + ttm_bo_get(bo); + mmap_read_unlock(vmf->vma->vm_mm); + if (!dma_resv_lock_interruptible(bo->base.resv, + NULL)) + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); + } + + return VM_FAULT_RETRY; + } + + if (dma_resv_lock_interruptible(bo->base.resv, NULL)) + return VM_FAULT_NOPAGE; + } + + return 0; +} +EXPORT_SYMBOL(ttm_bo_vm_reserve); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +/** + * ttm_bo_vm_insert_huge - Insert a pfn for PUD or PMD faults + * @vmf: Fault data + * @bo: The buffer object + * @page_offset: Page offset from bo start + * @fault_page_size: The size of the fault in pages. + * @pgprot: The page protections. + * Does additional checking whether it's possible to insert a PUD or PMD + * pfn and performs the insertion. + * + * Return: VM_FAULT_NOPAGE on successful insertion, VM_FAULT_FALLBACK if + * a huge fault was not possible, or on insertion error. + */ +static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf, + struct ttm_buffer_object *bo, + pgoff_t page_offset, + pgoff_t fault_page_size, + pgprot_t pgprot) +{ + pgoff_t i; + vm_fault_t ret; + unsigned long pfn; + pfn_t pfnt; + struct ttm_tt *ttm = bo->ttm; + bool write = vmf->flags & FAULT_FLAG_WRITE; + + /* Fault should not cross bo boundary. */ + page_offset &= ~(fault_page_size - 1); + if (page_offset + fault_page_size > bo->num_pages) + goto out_fallback; + + if (bo->mem.bus.is_iomem) + pfn = ttm_bo_io_mem_pfn(bo, page_offset); + else + pfn = page_to_pfn(ttm->pages[page_offset]); + + /* pfn must be fault_page_size aligned. */ + if ((pfn & (fault_page_size - 1)) != 0) + goto out_fallback; + + /* Check that memory is contiguous. */ + if (!bo->mem.bus.is_iomem) { + for (i = 1; i < fault_page_size; ++i) { + if (page_to_pfn(ttm->pages[page_offset + i]) != pfn + i) + goto out_fallback; + } + } else if (bo->bdev->driver->io_mem_pfn) { + for (i = 1; i < fault_page_size; ++i) { + if (ttm_bo_io_mem_pfn(bo, page_offset + i) != pfn + i) + goto out_fallback; + } + } + + pfnt = __pfn_to_pfn_t(pfn, PFN_DEV); + if (fault_page_size == (HPAGE_PMD_SIZE >> PAGE_SHIFT)) + ret = vmf_insert_pfn_pmd_prot(vmf, pfnt, pgprot, write); +#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD + else if (fault_page_size == (HPAGE_PUD_SIZE >> PAGE_SHIFT)) + ret = vmf_insert_pfn_pud_prot(vmf, pfnt, pgprot, write); +#endif + else + WARN_ON_ONCE(ret = VM_FAULT_FALLBACK); + + if (ret != VM_FAULT_NOPAGE) + goto out_fallback; + + return VM_FAULT_NOPAGE; +out_fallback: + count_vm_event(THP_FAULT_FALLBACK); + return VM_FAULT_FALLBACK; +} +#else +static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf, + struct ttm_buffer_object *bo, + pgoff_t page_offset, + pgoff_t fault_page_size, + pgprot_t pgprot) +{ + return VM_FAULT_FALLBACK; +} +#endif + +/** + * ttm_bo_vm_fault_reserved - TTM fault helper + * @vmf: The struct vm_fault given as argument to the fault callback + * @prot: The page protection to be used for this memory area. + * @num_prefault: Maximum number of prefault pages. The caller may want to + * specify this based on madvice settings and the size of the GPU object + * backed by the memory. + * @fault_page_size: The size of the fault in pages. + * + * This function inserts one or more page table entries pointing to the + * memory backing the buffer object, and then returns a return code + * instructing the caller to retry the page access. + * + * Return: + * VM_FAULT_NOPAGE on success or pending signal + * VM_FAULT_SIGBUS on unspecified error + * VM_FAULT_OOM on out-of-memory + * VM_FAULT_RETRY if retryable wait + */ +vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, + pgprot_t prot, + pgoff_t num_prefault, + pgoff_t fault_page_size) +{ + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = vma->vm_private_data; + struct ttm_bo_device *bdev = bo->bdev; + unsigned long page_offset; + unsigned long page_last; + unsigned long pfn; + struct ttm_tt *ttm = NULL; + struct page *page; + int err; + pgoff_t i; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long address = vmf->address; + + /* + * Refuse to fault imported pages. This should be handled + * (if at all) by redirecting mmap to the exporter. + */ + if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) + return VM_FAULT_SIGBUS; + + if (bdev->driver->fault_reserve_notify) { + struct dma_fence *moving = dma_fence_get(bo->moving); + + err = bdev->driver->fault_reserve_notify(bo); + switch (err) { + case 0: + break; + case -EBUSY: + case -ERESTARTSYS: + dma_fence_put(moving); + return VM_FAULT_NOPAGE; + default: + dma_fence_put(moving); + return VM_FAULT_SIGBUS; + } + + if (bo->moving != moving) { + ttm_bo_move_to_lru_tail_unlocked(bo); + } + dma_fence_put(moving); + } + + /* + * Wait for buffer data in transit, due to a pipelined + * move. + */ + ret = ttm_bo_vm_fault_idle(bo, vmf); + if (unlikely(ret != 0)) + return ret; + + err = ttm_mem_io_reserve(bdev, &bo->mem); + if (unlikely(err != 0)) + return VM_FAULT_SIGBUS; + + page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + + vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); + page_last = vma_pages(vma) + vma->vm_pgoff - + drm_vma_node_start(&bo->base.vma_node); + + if (unlikely(page_offset >= bo->num_pages)) + return VM_FAULT_SIGBUS; + + prot = ttm_io_prot(bo->mem.placement, prot); + if (!bo->mem.bus.is_iomem) { + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false, + .flags = TTM_OPT_FLAG_FORCE_ALLOC + + }; + + ttm = bo->ttm; + if (ttm_tt_populate(bdev, bo->ttm, &ctx)) + return VM_FAULT_OOM; + } else { + /* Iomem should not be marked encrypted */ + prot = pgprot_decrypted(prot); + } + + /* We don't prefault on huge faults. Yet. */ + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && fault_page_size != 1) + return ttm_bo_vm_insert_huge(vmf, bo, page_offset, + fault_page_size, prot); + + /* + * Speculatively prefault a number of pages. Only error on + * first page. + */ + for (i = 0; i < num_prefault; ++i) { + if (bo->mem.bus.is_iomem) { + pfn = ttm_bo_io_mem_pfn(bo, page_offset); + } else { + page = ttm->pages[page_offset]; + if (unlikely(!page && i == 0)) { + return VM_FAULT_OOM; + } else if (unlikely(!page)) { + break; + } + page->index = drm_vma_node_start(&bo->base.vma_node) + + page_offset; + pfn = page_to_pfn(page); + } + + /* + * Note that the value of @prot at this point may differ from + * the value of @vma->vm_page_prot in the caching- and + * encryption bits. This is because the exact location of the + * data may not be known at mmap() time and may also change + * at arbitrary times while the data is mmap'ed. + * See vmf_insert_mixed_prot() for a discussion. + */ + if (vma->vm_flags & VM_MIXEDMAP) + ret = vmf_insert_mixed_prot(vma, address, + __pfn_to_pfn_t(pfn, PFN_DEV), + prot); + else + ret = vmf_insert_pfn_prot(vma, address, pfn, prot); + + /* Never error on prefaulted PTEs */ + if (unlikely((ret & VM_FAULT_ERROR))) { + if (i == 0) + return VM_FAULT_NOPAGE; + else + break; + } + + address += PAGE_SIZE; + if (unlikely(++page_offset >= page_last)) + break; + } + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_fault_reserved); + +vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + pgprot_t prot; + struct ttm_buffer_object *bo = vma->vm_private_data; + vm_fault_t ret; + + ret = ttm_bo_vm_reserve(bo, vmf); + if (ret) + return ret; + + prot = vma->vm_page_prot; + ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return ret; + + dma_resv_unlock(bo->base.resv); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_fault); + +void ttm_bo_vm_open(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + + WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); + + ttm_bo_get(bo); +} +EXPORT_SYMBOL(ttm_bo_vm_open); + +void ttm_bo_vm_close(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + + ttm_bo_put(bo); + vma->vm_private_data = NULL; +} +EXPORT_SYMBOL(ttm_bo_vm_close); + +static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo, + unsigned long offset, + uint8_t *buf, int len, int write) +{ + unsigned long page = offset >> PAGE_SHIFT; + unsigned long bytes_left = len; + int ret; + + /* Copy a page at a time, that way no extra virtual address + * mapping is needed + */ + offset -= page << PAGE_SHIFT; + do { + unsigned long bytes = min(bytes_left, PAGE_SIZE - offset); + struct ttm_bo_kmap_obj map; + void *ptr; + bool is_iomem; + + ret = ttm_bo_kmap(bo, page, 1, &map); + if (ret) + return ret; + + ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset; + WARN_ON_ONCE(is_iomem); + if (write) + memcpy(ptr, buf, bytes); + else + memcpy(buf, ptr, bytes); + ttm_bo_kunmap(&map); + + page++; + buf += bytes; + bytes_left -= bytes; + offset = 0; + } while (bytes_left); + + return len; +} + +int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, + void *buf, int len, int write) +{ + struct ttm_buffer_object *bo = vma->vm_private_data; + unsigned long offset = (addr) - vma->vm_start + + ((vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node)) + << PAGE_SHIFT); + int ret; + + if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->num_pages) + return -EIO; + + ret = ttm_bo_reserve(bo, true, false, NULL); + if (ret) + return ret; + + switch (bo->mem.mem_type) { + case TTM_PL_SYSTEM: + fallthrough; + case TTM_PL_TT: + ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write); + break; + default: + if (bo->bdev->driver->access_memory) + ret = bo->bdev->driver->access_memory( + bo, offset, buf, len, write); + else + ret = -EIO; + } + + ttm_bo_unreserve(bo); + + return ret; +} +EXPORT_SYMBOL(ttm_bo_vm_access); + +static const struct vm_operations_struct ttm_bo_vm_ops = { + .fault = ttm_bo_vm_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close, + .access = ttm_bo_vm_access, +}; + +static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, + unsigned long offset, + unsigned long pages) +{ + struct drm_vma_offset_node *node; + struct ttm_buffer_object *bo = NULL; + + drm_vma_offset_lock_lookup(bdev->vma_manager); + + node = drm_vma_offset_lookup_locked(bdev->vma_manager, offset, pages); + if (likely(node)) { + bo = container_of(node, struct ttm_buffer_object, + base.vma_node); + bo = ttm_bo_get_unless_zero(bo); + } + + drm_vma_offset_unlock_lookup(bdev->vma_manager); + + if (!bo) + pr_err("Could not find buffer object to map\n"); + + return bo; +} + +static void ttm_bo_mmap_vma_setup(struct ttm_buffer_object *bo, struct vm_area_struct *vma) +{ + vma->vm_ops = &ttm_bo_vm_ops; + + /* + * Note: We're transferring the bo reference to + * vma->vm_private_data here. + */ + + vma->vm_private_data = bo; + + /* + * We'd like to use VM_PFNMAP on shared mappings, where + * (vma->vm_flags & VM_SHARED) != 0, for performance reasons, + * but for some reason VM_PFNMAP + x86 PAT + write-combine is very + * bad for performance. Until that has been sorted out, use + * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 + */ + vma->vm_flags |= VM_MIXEDMAP; + vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; +} + +int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, + struct ttm_bo_device *bdev) +{ + struct ttm_bo_driver *driver; + struct ttm_buffer_object *bo; + int ret; + + if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET_START)) + return -EINVAL; + + bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); + if (unlikely(!bo)) + return -EINVAL; + + driver = bo->bdev->driver; + if (unlikely(!driver->verify_access)) { + ret = -EPERM; + goto out_unref; + } + ret = driver->verify_access(bo, filp); + if (unlikely(ret != 0)) + goto out_unref; + + ttm_bo_mmap_vma_setup(bo, vma); + return 0; +out_unref: + ttm_bo_put(bo); + return ret; +} +EXPORT_SYMBOL(ttm_bo_mmap); + +int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) +{ + ttm_bo_get(bo); + ttm_bo_mmap_vma_setup(bo, vma); + return 0; +} +EXPORT_SYMBOL(ttm_bo_mmap_obj); diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c new file mode 100644 index 000000000..8a8f1a6a8 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#include +#include +#include +#include +#include +#include + +static void ttm_eu_backoff_reservation_reverse(struct list_head *list, + struct ttm_validate_buffer *entry) +{ + list_for_each_entry_continue_reverse(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + dma_resv_unlock(bo->base.resv); + } +} + +void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, + struct list_head *list) +{ + struct ttm_validate_buffer *entry; + + if (list_empty(list)) + return; + + spin_lock(&ttm_bo_glob.lru_lock); + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + ttm_bo_move_to_lru_tail(bo, NULL); + dma_resv_unlock(bo->base.resv); + } + spin_unlock(&ttm_bo_glob.lru_lock); + + if (ticket) + ww_acquire_fini(ticket); +} +EXPORT_SYMBOL(ttm_eu_backoff_reservation); + +/* + * Reserve buffers for validation. + * + * If a buffer in the list is marked for CPU access, we back off and + * wait for that buffer to become free for GPU access. + * + * If a buffer is reserved for another validation, the validator with + * the highest validation sequence backs off and waits for that buffer + * to become unreserved. This prevents deadlocks when validating multiple + * buffers in different orders. + */ + +int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, + struct list_head *list, bool intr, + struct list_head *dups) +{ + struct ttm_validate_buffer *entry; + int ret; + + if (list_empty(list)) + return 0; + + if (ticket) + ww_acquire_init(ticket, &reservation_ww_class); + + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + ret = ttm_bo_reserve(bo, intr, (ticket == NULL), ticket); + if (ret == -EALREADY && dups) { + struct ttm_validate_buffer *safe = entry; + entry = list_prev_entry(entry, head); + list_del(&safe->head); + list_add(&safe->head, dups); + continue; + } + + if (!ret) { + if (!entry->num_shared) + continue; + + ret = dma_resv_reserve_shared(bo->base.resv, + entry->num_shared); + if (!ret) + continue; + } + + /* uh oh, we lost out, drop every reservation and try + * to only reserve this buffer, then start over if + * this succeeds. + */ + ttm_eu_backoff_reservation_reverse(list, entry); + + if (ret == -EDEADLK) { + ret = ttm_bo_reserve_slowpath(bo, intr, ticket); + } + + if (!ret && entry->num_shared) + ret = dma_resv_reserve_shared(bo->base.resv, + entry->num_shared); + + if (unlikely(ret != 0)) { + if (ticket) { + ww_acquire_done(ticket); + ww_acquire_fini(ticket); + } + return ret; + } + + /* move this item to the front of the list, + * forces correct iteration of the loop without keeping track + */ + list_del(&entry->head); + list_add(&entry->head, list); + } + + return 0; +} +EXPORT_SYMBOL(ttm_eu_reserve_buffers); + +void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, + struct list_head *list, + struct dma_fence *fence) +{ + struct ttm_validate_buffer *entry; + + if (list_empty(list)) + return; + + spin_lock(&ttm_bo_glob.lru_lock); + list_for_each_entry(entry, list, head) { + struct ttm_buffer_object *bo = entry->bo; + + if (entry->num_shared) + dma_resv_add_shared_fence(bo->base.resv, fence); + else + dma_resv_add_excl_fence(bo->base.resv, fence); + ttm_bo_move_to_lru_tail(bo, NULL); + dma_resv_unlock(bo->base.resv); + } + spin_unlock(&ttm_bo_glob.lru_lock); + if (ticket) + ww_acquire_fini(ticket); +} +EXPORT_SYMBOL(ttm_eu_fence_buffer_objects); diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c new file mode 100644 index 000000000..89d50f38c --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -0,0 +1,683 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TTM_MEMORY_ALLOC_RETRIES 4 + +struct ttm_mem_global ttm_mem_glob; +EXPORT_SYMBOL(ttm_mem_glob); + +struct ttm_mem_zone { + struct kobject kobj; + struct ttm_mem_global *glob; + const char *name; + uint64_t zone_mem; + uint64_t emer_mem; + uint64_t max_mem; + uint64_t swap_limit; + uint64_t used_mem; +}; + +static struct attribute ttm_mem_sys = { + .name = "zone_memory", + .mode = S_IRUGO +}; +static struct attribute ttm_mem_emer = { + .name = "emergency_memory", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_mem_max = { + .name = "available_memory", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_mem_swap = { + .name = "swap_limit", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_mem_used = { + .name = "used_memory", + .mode = S_IRUGO +}; + +static void ttm_mem_zone_kobj_release(struct kobject *kobj) +{ + struct ttm_mem_zone *zone = + container_of(kobj, struct ttm_mem_zone, kobj); + + pr_info("Zone %7s: Used memory at exit: %llu KiB\n", + zone->name, (unsigned long long)zone->used_mem >> 10); + kfree(zone); +} + +static ssize_t ttm_mem_zone_show(struct kobject *kobj, + struct attribute *attr, + char *buffer) +{ + struct ttm_mem_zone *zone = + container_of(kobj, struct ttm_mem_zone, kobj); + uint64_t val = 0; + + spin_lock(&zone->glob->lock); + if (attr == &ttm_mem_sys) + val = zone->zone_mem; + else if (attr == &ttm_mem_emer) + val = zone->emer_mem; + else if (attr == &ttm_mem_max) + val = zone->max_mem; + else if (attr == &ttm_mem_swap) + val = zone->swap_limit; + else if (attr == &ttm_mem_used) + val = zone->used_mem; + spin_unlock(&zone->glob->lock); + + return snprintf(buffer, PAGE_SIZE, "%llu\n", + (unsigned long long) val >> 10); +} + +static void ttm_check_swapping(struct ttm_mem_global *glob); + +static ssize_t ttm_mem_zone_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, + size_t size) +{ + struct ttm_mem_zone *zone = + container_of(kobj, struct ttm_mem_zone, kobj); + int chars; + unsigned long val; + uint64_t val64; + + chars = sscanf(buffer, "%lu", &val); + if (chars == 0) + return size; + + val64 = val; + val64 <<= 10; + + spin_lock(&zone->glob->lock); + if (val64 > zone->zone_mem) + val64 = zone->zone_mem; + if (attr == &ttm_mem_emer) { + zone->emer_mem = val64; + if (zone->max_mem > val64) + zone->max_mem = val64; + } else if (attr == &ttm_mem_max) { + zone->max_mem = val64; + if (zone->emer_mem < val64) + zone->emer_mem = val64; + } else if (attr == &ttm_mem_swap) + zone->swap_limit = val64; + spin_unlock(&zone->glob->lock); + + ttm_check_swapping(zone->glob); + + return size; +} + +static struct attribute *ttm_mem_zone_attrs[] = { + &ttm_mem_sys, + &ttm_mem_emer, + &ttm_mem_max, + &ttm_mem_swap, + &ttm_mem_used, + NULL +}; + +static const struct sysfs_ops ttm_mem_zone_ops = { + .show = &ttm_mem_zone_show, + .store = &ttm_mem_zone_store +}; + +static struct kobj_type ttm_mem_zone_kobj_type = { + .release = &ttm_mem_zone_kobj_release, + .sysfs_ops = &ttm_mem_zone_ops, + .default_attrs = ttm_mem_zone_attrs, +}; + +static struct attribute ttm_mem_global_lower_mem_limit = { + .name = "lower_mem_limit", + .mode = S_IRUGO | S_IWUSR +}; + +static ssize_t ttm_mem_global_show(struct kobject *kobj, + struct attribute *attr, + char *buffer) +{ + struct ttm_mem_global *glob = + container_of(kobj, struct ttm_mem_global, kobj); + uint64_t val = 0; + + spin_lock(&glob->lock); + val = glob->lower_mem_limit; + spin_unlock(&glob->lock); + /* convert from number of pages to KB */ + val <<= (PAGE_SHIFT - 10); + return snprintf(buffer, PAGE_SIZE, "%llu\n", + (unsigned long long) val); +} + +static ssize_t ttm_mem_global_store(struct kobject *kobj, + struct attribute *attr, + const char *buffer, + size_t size) +{ + int chars; + uint64_t val64; + unsigned long val; + struct ttm_mem_global *glob = + container_of(kobj, struct ttm_mem_global, kobj); + + chars = sscanf(buffer, "%lu", &val); + if (chars == 0) + return size; + + val64 = val; + /* convert from KB to number of pages */ + val64 >>= (PAGE_SHIFT - 10); + + spin_lock(&glob->lock); + glob->lower_mem_limit = val64; + spin_unlock(&glob->lock); + + return size; +} + +static struct attribute *ttm_mem_global_attrs[] = { + &ttm_mem_global_lower_mem_limit, + NULL +}; + +static const struct sysfs_ops ttm_mem_global_ops = { + .show = &ttm_mem_global_show, + .store = &ttm_mem_global_store, +}; + +static struct kobj_type ttm_mem_glob_kobj_type = { + .sysfs_ops = &ttm_mem_global_ops, + .default_attrs = ttm_mem_global_attrs, +}; + +static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, + bool from_wq, uint64_t extra) +{ + unsigned int i; + struct ttm_mem_zone *zone; + uint64_t target; + + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + + if (from_wq) + target = zone->swap_limit; + else if (capable(CAP_SYS_ADMIN)) + target = zone->emer_mem; + else + target = zone->max_mem; + + target = (extra > target) ? 0ULL : target; + + if (zone->used_mem > target) + return true; + } + return false; +} + +/* + * At this point we only support a single shrink callback. + * Extend this if needed, perhaps using a linked list of callbacks. + * Note that this function is reentrant: + * many threads may try to swap out at any given time. + */ + +static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, + uint64_t extra, struct ttm_operation_ctx *ctx) +{ + int ret; + + spin_lock(&glob->lock); + + while (ttm_zones_above_swap_target(glob, from_wq, extra)) { + spin_unlock(&glob->lock); + ret = ttm_bo_swapout(&ttm_bo_glob, ctx); + spin_lock(&glob->lock); + if (unlikely(ret != 0)) + break; + } + + spin_unlock(&glob->lock); +} + +static void ttm_shrink_work(struct work_struct *work) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + struct ttm_mem_global *glob = + container_of(work, struct ttm_mem_global, work); + + ttm_shrink(glob, true, 0ULL, &ctx); +} + +static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, + const struct sysinfo *si) +{ + struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); + uint64_t mem; + int ret; + + if (unlikely(!zone)) + return -ENOMEM; + + mem = si->totalram - si->totalhigh; + mem *= si->mem_unit; + + zone->name = "kernel"; + zone->zone_mem = mem; + zone->max_mem = mem >> 1; + zone->emer_mem = (mem >> 1) + (mem >> 2); + zone->swap_limit = zone->max_mem - (mem >> 3); + zone->used_mem = 0; + zone->glob = glob; + glob->zone_kernel = zone; + ret = kobject_init_and_add( + &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); + if (unlikely(ret != 0)) { + kobject_put(&zone->kobj); + return ret; + } + glob->zones[glob->num_zones++] = zone; + return 0; +} + +#ifdef CONFIG_HIGHMEM +static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob, + const struct sysinfo *si) +{ + struct ttm_mem_zone *zone; + uint64_t mem; + int ret; + + if (si->totalhigh == 0) + return 0; + + zone = kzalloc(sizeof(*zone), GFP_KERNEL); + if (unlikely(!zone)) + return -ENOMEM; + + mem = si->totalram; + mem *= si->mem_unit; + + zone->name = "highmem"; + zone->zone_mem = mem; + zone->max_mem = mem >> 1; + zone->emer_mem = (mem >> 1) + (mem >> 2); + zone->swap_limit = zone->max_mem - (mem >> 3); + zone->used_mem = 0; + zone->glob = glob; + glob->zone_highmem = zone; + ret = kobject_init_and_add( + &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s", + zone->name); + if (unlikely(ret != 0)) { + kobject_put(&zone->kobj); + return ret; + } + glob->zones[glob->num_zones++] = zone; + return 0; +} +#else +static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, + const struct sysinfo *si) +{ + struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); + uint64_t mem; + int ret; + + if (unlikely(!zone)) + return -ENOMEM; + + mem = si->totalram; + mem *= si->mem_unit; + + /** + * No special dma32 zone needed. + */ + + if (mem <= ((uint64_t) 1ULL << 32)) { + kfree(zone); + return 0; + } + + /* + * Limit max dma32 memory to 4GB for now + * until we can figure out how big this + * zone really is. + */ + + mem = ((uint64_t) 1ULL << 32); + zone->name = "dma32"; + zone->zone_mem = mem; + zone->max_mem = mem >> 1; + zone->emer_mem = (mem >> 1) + (mem >> 2); + zone->swap_limit = zone->max_mem - (mem >> 3); + zone->used_mem = 0; + zone->glob = glob; + glob->zone_dma32 = zone; + ret = kobject_init_and_add( + &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); + if (unlikely(ret != 0)) { + kobject_put(&zone->kobj); + return ret; + } + glob->zones[glob->num_zones++] = zone; + return 0; +} +#endif + +int ttm_mem_global_init(struct ttm_mem_global *glob) +{ + struct sysinfo si; + int ret; + int i; + struct ttm_mem_zone *zone; + + spin_lock_init(&glob->lock); + glob->swap_queue = create_singlethread_workqueue("ttm_swap"); + INIT_WORK(&glob->work, ttm_shrink_work); + ret = kobject_init_and_add( + &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting"); + if (unlikely(ret != 0)) { + kobject_put(&glob->kobj); + return ret; + } + + si_meminfo(&si); + + /* set it as 0 by default to keep original behavior of OOM */ + glob->lower_mem_limit = 0; + + ret = ttm_mem_init_kernel_zone(glob, &si); + if (unlikely(ret != 0)) + goto out_no_zone; +#ifdef CONFIG_HIGHMEM + ret = ttm_mem_init_highmem_zone(glob, &si); + if (unlikely(ret != 0)) + goto out_no_zone; +#else + ret = ttm_mem_init_dma32_zone(glob, &si); + if (unlikely(ret != 0)) + goto out_no_zone; +#endif + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + pr_info("Zone %7s: Available graphics memory: %llu KiB\n", + zone->name, (unsigned long long)zone->max_mem >> 10); + } + ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); + ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); + return 0; +out_no_zone: + ttm_mem_global_release(glob); + return ret; +} + +void ttm_mem_global_release(struct ttm_mem_global *glob) +{ + struct ttm_mem_zone *zone; + unsigned int i; + + /* let the page allocator first stop the shrink work. */ + ttm_page_alloc_fini(); + ttm_dma_page_alloc_fini(); + + flush_workqueue(glob->swap_queue); + destroy_workqueue(glob->swap_queue); + glob->swap_queue = NULL; + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + kobject_del(&zone->kobj); + kobject_put(&zone->kobj); + } + kobject_del(&glob->kobj); + kobject_put(&glob->kobj); + memset(glob, 0, sizeof(*glob)); +} + +static void ttm_check_swapping(struct ttm_mem_global *glob) +{ + bool needs_swapping = false; + unsigned int i; + struct ttm_mem_zone *zone; + + spin_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (zone->used_mem > zone->swap_limit) { + needs_swapping = true; + break; + } + } + + spin_unlock(&glob->lock); + + if (unlikely(needs_swapping)) + (void)queue_work(glob->swap_queue, &glob->work); + +} + +static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t amount) +{ + unsigned int i; + struct ttm_mem_zone *zone; + + spin_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + zone->used_mem -= amount; + } + spin_unlock(&glob->lock); +} + +void ttm_mem_global_free(struct ttm_mem_global *glob, + uint64_t amount) +{ + return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount); +} +EXPORT_SYMBOL(ttm_mem_global_free); + +/* + * check if the available mem is under lower memory limit + * + * a. if no swap disk at all or free swap space is under swap_mem_limit + * but available system mem is bigger than sys_mem_limit, allow TTM + * allocation; + * + * b. if the available system mem is less than sys_mem_limit but free + * swap disk is bigger than swap_mem_limit, allow TTM allocation. + */ +bool +ttm_check_under_lowerlimit(struct ttm_mem_global *glob, + uint64_t num_pages, + struct ttm_operation_ctx *ctx) +{ + int64_t available; + + if (ctx->flags & TTM_OPT_FLAG_FORCE_ALLOC) + return false; + + available = get_nr_swap_pages() + si_mem_available(); + available -= num_pages; + if (available < glob->lower_mem_limit) + return true; + + return false; +} + +static int ttm_mem_global_reserve(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t amount, bool reserve) +{ + uint64_t limit; + int ret = -ENOMEM; + unsigned int i; + struct ttm_mem_zone *zone; + + spin_lock(&glob->lock); + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + + limit = (capable(CAP_SYS_ADMIN)) ? + zone->emer_mem : zone->max_mem; + + if (zone->used_mem > limit) + goto out_unlock; + } + + if (reserve) { + for (i = 0; i < glob->num_zones; ++i) { + zone = glob->zones[i]; + if (single_zone && zone != single_zone) + continue; + zone->used_mem += amount; + } + } + + ret = 0; +out_unlock: + spin_unlock(&glob->lock); + ttm_check_swapping(glob); + + return ret; +} + + +static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, + struct ttm_mem_zone *single_zone, + uint64_t memory, + struct ttm_operation_ctx *ctx) +{ + int count = TTM_MEMORY_ALLOC_RETRIES; + + while (unlikely(ttm_mem_global_reserve(glob, + single_zone, + memory, true) + != 0)) { + if (ctx->no_wait_gpu) + return -ENOMEM; + if (unlikely(count-- == 0)) + return -ENOMEM; + ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx); + } + + return 0; +} + +int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, + struct ttm_operation_ctx *ctx) +{ + /** + * Normal allocations of kernel memory are registered in + * the kernel zone. + */ + + return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx); +} +EXPORT_SYMBOL(ttm_mem_global_alloc); + +int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, + struct page *page, uint64_t size, + struct ttm_operation_ctx *ctx) +{ + struct ttm_mem_zone *zone = NULL; + + /** + * Page allocations may be registed in a single zone + * only if highmem or !dma32. + */ + +#ifdef CONFIG_HIGHMEM + if (PageHighMem(page) && glob->zone_highmem != NULL) + zone = glob->zone_highmem; +#else + if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) + zone = glob->zone_kernel; +#endif + return ttm_mem_global_alloc_zone(glob, zone, size, ctx); +} + +void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, + uint64_t size) +{ + struct ttm_mem_zone *zone = NULL; + +#ifdef CONFIG_HIGHMEM + if (PageHighMem(page) && glob->zone_highmem != NULL) + zone = glob->zone_highmem; +#else + if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) + zone = glob->zone_kernel; +#endif + ttm_mem_global_free_zone(glob, zone, size); +} + +size_t ttm_round_pot(size_t size) +{ + if ((size & (size - 1)) == 0) + return size; + else if (size > PAGE_SIZE) + return PAGE_ALIGN(size); + else { + size_t tmp_size = 4; + + while (tmp_size < size) + tmp_size <<= 1; + + return tmp_size; + } + return 0; +} +EXPORT_SYMBOL(ttm_round_pot); diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c new file mode 100644 index 000000000..6ff40c041 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + * Jerome Glisse + */ +#include +#include +#include +#include +#include + +static DECLARE_WAIT_QUEUE_HEAD(exit_q); +static atomic_t device_released; + +static struct device_type ttm_drm_class_type = { + .name = "ttm", + /** + * Add pm ops here. + */ +}; + +static void ttm_drm_class_device_release(struct device *dev) +{ + atomic_set(&device_released, 1); + wake_up_all(&exit_q); +} + +static struct device ttm_drm_class_device = { + .type = &ttm_drm_class_type, + .release = &ttm_drm_class_device_release +}; + +struct kobject *ttm_get_kobj(void) +{ + struct kobject *kobj = &ttm_drm_class_device.kobj; + BUG_ON(kobj == NULL); + return kobj; +} + +static int __init ttm_init(void) +{ + int ret; + + ret = dev_set_name(&ttm_drm_class_device, "ttm"); + if (unlikely(ret != 0)) + return ret; + + atomic_set(&device_released, 0); + ret = drm_class_device_register(&ttm_drm_class_device); + if (unlikely(ret != 0)) + goto out_no_dev_reg; + + return 0; +out_no_dev_reg: + atomic_set(&device_released, 1); + wake_up_all(&exit_q); + return ret; +} + +static void __exit ttm_exit(void) +{ + drm_class_device_unregister(&ttm_drm_class_device); + + /** + * Refuse to unload until the TTM device is released. + * Not sure this is 100% needed. + */ + + wait_event(exit_q, atomic_read(&device_released) == 1); +} + +module_init(ttm_init); +module_exit(ttm_exit); + +MODULE_AUTHOR("Thomas Hellstrom, Jerome Glisse"); +MODULE_DESCRIPTION("TTM memory manager subsystem (for DRM device)"); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c new file mode 100644 index 000000000..14660f723 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -0,0 +1,1189 @@ +/* + * Copyright (c) Red Hat Inc. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie + * Jerome Glisse + * Pauli Nieminen + */ + +/* simple list based uncached page pool + * - Pool collects resently freed pages for reuse + * - Use page->lru to keep a free list + * - doesn't track currently in use pages + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include /* for seq_printf */ +#include +#include + +#include + +#include +#include +#include + +#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) +#define SMALL_ALLOCATION 16 +#define FREE_ALL_PAGES (~0U) +/* times are in msecs */ +#define PAGE_FREE_INTERVAL 1000 + +/** + * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. + * + * @lock: Protects the shared pool from concurrnet access. Must be used with + * irqsave/irqrestore variants because pool allocator maybe called from + * delayed work. + * @fill_lock: Prevent concurrent calls to fill. + * @list: Pool of free uc/wc pages for fast reuse. + * @gfp_flags: Flags to pass for alloc_page. + * @npages: Number of pages in pool. + */ +struct ttm_page_pool { + spinlock_t lock; + bool fill_lock; + struct list_head list; + gfp_t gfp_flags; + unsigned npages; + char *name; + unsigned long nfrees; + unsigned long nrefills; + unsigned int order; +}; + +/** + * Limits for the pool. They are handled without locks because only place where + * they may change is in sysfs store. They won't have immediate effect anyway + * so forcing serialization to access them is pointless. + */ + +struct ttm_pool_opts { + unsigned alloc_size; + unsigned max_size; + unsigned small; +}; + +#define NUM_POOLS 6 + +/** + * struct ttm_pool_manager - Holds memory pools for fst allocation + * + * Manager is read only object for pool code so it doesn't need locking. + * + * @free_interval: minimum number of jiffies between freeing pages from pool. + * @page_alloc_inited: reference counting for pool allocation. + * @work: Work that is used to shrink the pool. Work is only run when there is + * some pages to free. + * @small_allocation: Limit in number of pages what is small allocation. + * + * @pools: All pool objects in use. + **/ +struct ttm_pool_manager { + struct kobject kobj; + struct shrinker mm_shrink; + struct ttm_pool_opts options; + + union { + struct ttm_page_pool pools[NUM_POOLS]; + struct { + struct ttm_page_pool wc_pool; + struct ttm_page_pool uc_pool; + struct ttm_page_pool wc_pool_dma32; + struct ttm_page_pool uc_pool_dma32; + struct ttm_page_pool wc_pool_huge; + struct ttm_page_pool uc_pool_huge; + } ; + }; +}; + +static struct attribute ttm_page_pool_max = { + .name = "pool_max_size", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_small = { + .name = "pool_small_allocation", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_alloc_size = { + .name = "pool_allocation_size", + .mode = S_IRUGO | S_IWUSR +}; + +static struct attribute *ttm_pool_attrs[] = { + &ttm_page_pool_max, + &ttm_page_pool_small, + &ttm_page_pool_alloc_size, + NULL +}; + +static void ttm_pool_kobj_release(struct kobject *kobj) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + kfree(m); +} + +static ssize_t ttm_pool_store(struct kobject *kobj, + struct attribute *attr, const char *buffer, size_t size) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + int chars; + unsigned val; + chars = sscanf(buffer, "%u", &val); + if (chars == 0) + return size; + + /* Convert kb to number of pages */ + val = val / (PAGE_SIZE >> 10); + + if (attr == &ttm_page_pool_max) + m->options.max_size = val; + else if (attr == &ttm_page_pool_small) + m->options.small = val; + else if (attr == &ttm_page_pool_alloc_size) { + if (val > NUM_PAGES_TO_ALLOC*8) { + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + return size; + } else if (val > NUM_PAGES_TO_ALLOC) { + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + } + m->options.alloc_size = val; + } + + return size; +} + +static ssize_t ttm_pool_show(struct kobject *kobj, + struct attribute *attr, char *buffer) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + unsigned val = 0; + + if (attr == &ttm_page_pool_max) + val = m->options.max_size; + else if (attr == &ttm_page_pool_small) + val = m->options.small; + else if (attr == &ttm_page_pool_alloc_size) + val = m->options.alloc_size; + + val = val * (PAGE_SIZE >> 10); + + return snprintf(buffer, PAGE_SIZE, "%u\n", val); +} + +static const struct sysfs_ops ttm_pool_sysfs_ops = { + .show = &ttm_pool_show, + .store = &ttm_pool_store, +}; + +static struct kobj_type ttm_pool_kobj_type = { + .release = &ttm_pool_kobj_release, + .sysfs_ops = &ttm_pool_sysfs_ops, + .default_attrs = ttm_pool_attrs, +}; + +static struct ttm_pool_manager *_manager; + +/** + * Select the right pool or requested caching state and ttm flags. */ +static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, + enum ttm_caching_state cstate) +{ + int pool_index; + + if (cstate == tt_cached) + return NULL; + + if (cstate == tt_wc) + pool_index = 0x0; + else + pool_index = 0x1; + + if (flags & TTM_PAGE_FLAG_DMA32) { + if (huge) + return NULL; + pool_index |= 0x2; + + } else if (huge) { + pool_index |= 0x4; + } + + return &_manager->pools[pool_index]; +} + +/* set memory back to wb and free the pages. */ +static void ttm_pages_put(struct page *pages[], unsigned npages, + unsigned int order) +{ + unsigned int i, pages_nr = (1 << order); + + if (order == 0) { + if (ttm_set_pages_array_wb(pages, npages)) + pr_err("Failed to set %d pages to wb!\n", npages); + } + + for (i = 0; i < npages; ++i) { + if (order > 0) { + if (ttm_set_pages_wb(pages[i], pages_nr)) + pr_err("Failed to set %d pages to wb!\n", pages_nr); + } + __free_pages(pages[i], order); + } +} + +static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, + unsigned freed_pages) +{ + pool->npages -= freed_pages; + pool->nfrees += freed_pages; +} + +/** + * Free pages from pool. + * + * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC + * number of pages in one go. + * + * @pool: to free the pages from + * @free_all: If set to true will free all pages in pool + * @use_static: Safe to use static buffer + **/ +static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, + bool use_static) +{ + static struct page *static_buf[NUM_PAGES_TO_ALLOC]; + unsigned long irq_flags; + struct page *p; + struct page **pages_to_free; + unsigned freed_pages = 0, + npages_to_free = nr_free; + + if (NUM_PAGES_TO_ALLOC < nr_free) + npages_to_free = NUM_PAGES_TO_ALLOC; + + if (use_static) + pages_to_free = static_buf; + else + pages_to_free = kmalloc_array(npages_to_free, + sizeof(struct page *), + GFP_KERNEL); + if (!pages_to_free) { + pr_debug("Failed to allocate memory for pool free operation\n"); + return 0; + } + +restart: + spin_lock_irqsave(&pool->lock, irq_flags); + + list_for_each_entry_reverse(p, &pool->list, lru) { + if (freed_pages >= npages_to_free) + break; + + pages_to_free[freed_pages++] = p; + /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ + if (freed_pages >= NUM_PAGES_TO_ALLOC) { + /* remove range of pages from the pool */ + __list_del(p->lru.prev, &pool->list); + + ttm_pool_update_free_locked(pool, freed_pages); + /** + * Because changing page caching is costly + * we unlock the pool to prevent stalling. + */ + spin_unlock_irqrestore(&pool->lock, irq_flags); + + ttm_pages_put(pages_to_free, freed_pages, pool->order); + if (likely(nr_free != FREE_ALL_PAGES)) + nr_free -= freed_pages; + + if (NUM_PAGES_TO_ALLOC >= nr_free) + npages_to_free = nr_free; + else + npages_to_free = NUM_PAGES_TO_ALLOC; + + freed_pages = 0; + + /* free all so restart the processing */ + if (nr_free) + goto restart; + + /* Not allowed to fall through or break because + * following context is inside spinlock while we are + * outside here. + */ + goto out; + + } + } + + /* remove range of pages from the pool */ + if (freed_pages) { + __list_del(&p->lru, &pool->list); + + ttm_pool_update_free_locked(pool, freed_pages); + nr_free -= freed_pages; + } + + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (freed_pages) + ttm_pages_put(pages_to_free, freed_pages, pool->order); +out: + if (pages_to_free != static_buf) + kfree(pages_to_free); + return nr_free; +} + +/** + * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * This code is crying out for a shrinker per pool.... + */ +static unsigned long +ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + static DEFINE_MUTEX(lock); + static unsigned start_pool; + unsigned i; + unsigned pool_offset; + struct ttm_page_pool *pool; + int shrink_pages = sc->nr_to_scan; + unsigned long freed = 0; + unsigned int nr_free_pool; + + if (!mutex_trylock(&lock)) + return SHRINK_STOP; + pool_offset = ++start_pool % NUM_POOLS; + /* select start pool in round robin fashion */ + for (i = 0; i < NUM_POOLS; ++i) { + unsigned nr_free = shrink_pages; + unsigned page_nr; + + if (shrink_pages == 0) + break; + + pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; + page_nr = (1 << pool->order); + /* OK to use static buffer since global mutex is held. */ + nr_free_pool = roundup(nr_free, page_nr) >> pool->order; + shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true); + freed += (nr_free_pool - shrink_pages) << pool->order; + if (freed >= sc->nr_to_scan) + break; + shrink_pages <<= pool->order; + } + mutex_unlock(&lock); + return freed; +} + + +static unsigned long +ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned i; + unsigned long count = 0; + struct ttm_page_pool *pool; + + for (i = 0; i < NUM_POOLS; ++i) { + pool = &_manager->pools[i]; + count += (pool->npages << pool->order); + } + + return count; +} + +static int ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) +{ + manager->mm_shrink.count_objects = ttm_pool_shrink_count; + manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; + manager->mm_shrink.seeks = 1; + return register_shrinker(&manager->mm_shrink); +} + +static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +{ + unregister_shrinker(&manager->mm_shrink); +} + +static int ttm_set_pages_caching(struct page **pages, + enum ttm_caching_state cstate, unsigned cpages) +{ + int r = 0; + /* Set page caching */ + switch (cstate) { + case tt_uncached: + r = ttm_set_pages_array_uc(pages, cpages); + if (r) + pr_err("Failed to set %d pages to uc!\n", cpages); + break; + case tt_wc: + r = ttm_set_pages_array_wc(pages, cpages); + if (r) + pr_err("Failed to set %d pages to wc!\n", cpages); + break; + default: + break; + } + return r; +} + +/** + * Free pages the pages that failed to change the caching state. If there is + * any pages that have changed their caching state already put them to the + * pool. + */ +static void ttm_handle_caching_state_failure(struct list_head *pages, + int ttm_flags, enum ttm_caching_state cstate, + struct page **failed_pages, unsigned cpages) +{ + unsigned i; + /* Failed pages have to be freed */ + for (i = 0; i < cpages; ++i) { + list_del(&failed_pages[i]->lru); + __free_page(failed_pages[i]); + } +} + +/** + * Allocate new pages with correct caching. + * + * This function is reentrant if caller updates count depending on number of + * pages returned in pages array. + */ +static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, + int ttm_flags, enum ttm_caching_state cstate, + unsigned count, unsigned order) +{ + struct page **caching_array; + struct page *p; + int r = 0; + unsigned i, j, cpages; + unsigned npages = 1 << order; + unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); + + /* allocate array for page caching change */ + caching_array = kmalloc_array(max_cpages, sizeof(struct page *), + GFP_KERNEL); + + if (!caching_array) { + pr_debug("Unable to allocate table for new pages\n"); + return -ENOMEM; + } + + for (i = 0, cpages = 0; i < count; ++i) { + p = alloc_pages(gfp_flags, order); + + if (!p) { + pr_debug("Unable to get page %u\n", i); + + /* store already allocated pages in the pool after + * setting the caching state */ + if (cpages) { + r = ttm_set_pages_caching(caching_array, + cstate, cpages); + if (r) + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + } + r = -ENOMEM; + goto out; + } + + list_add(&p->lru, pages); + +#ifdef CONFIG_HIGHMEM + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ + if (PageHighMem(p)) + continue; + +#endif + for (j = 0; j < npages; ++j) { + caching_array[cpages++] = p++; + if (cpages == max_cpages) { + + r = ttm_set_pages_caching(caching_array, + cstate, cpages); + if (r) { + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + goto out; + } + cpages = 0; + } + } + } + + if (cpages) { + r = ttm_set_pages_caching(caching_array, cstate, cpages); + if (r) + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + } +out: + kfree(caching_array); + + return r; +} + +/** + * Fill the given pool if there aren't enough pages and the requested number of + * pages is small. + */ +static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, + enum ttm_caching_state cstate, + unsigned count, unsigned long *irq_flags) +{ + struct page *p; + int r; + unsigned cpages = 0; + /** + * Only allow one pool fill operation at a time. + * If pool doesn't have enough pages for the allocation new pages are + * allocated from outside of pool. + */ + if (pool->fill_lock) + return; + + pool->fill_lock = true; + + /* If allocation request is small and there are not enough + * pages in a pool we fill the pool up first. */ + if (count < _manager->options.small + && count > pool->npages) { + struct list_head new_pages; + unsigned alloc_size = _manager->options.alloc_size; + + /** + * Can't change page caching if in irqsave context. We have to + * drop the pool->lock. + */ + spin_unlock_irqrestore(&pool->lock, *irq_flags); + + INIT_LIST_HEAD(&new_pages); + r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, + cstate, alloc_size, 0); + spin_lock_irqsave(&pool->lock, *irq_flags); + + if (!r) { + list_splice(&new_pages, &pool->list); + ++pool->nrefills; + pool->npages += alloc_size; + } else { + pr_debug("Failed to fill pool (%p)\n", pool); + /* If we have any pages left put them to the pool. */ + list_for_each_entry(p, &new_pages, lru) { + ++cpages; + } + list_splice(&new_pages, &pool->list); + pool->npages += cpages; + } + + } + pool->fill_lock = false; +} + +/** + * Allocate pages from the pool and put them on the return list. + * + * @return zero for success or negative error code. + */ +static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, + struct list_head *pages, + int ttm_flags, + enum ttm_caching_state cstate, + unsigned count, unsigned order) +{ + unsigned long irq_flags; + struct list_head *p; + unsigned i; + int r = 0; + + spin_lock_irqsave(&pool->lock, irq_flags); + if (!order) + ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, + &irq_flags); + + if (count >= pool->npages) { + /* take all pages from the pool */ + list_splice_init(&pool->list, pages); + count -= pool->npages; + pool->npages = 0; + goto out; + } + /* find the last pages to include for requested number of pages. Split + * pool to begin and halve it to reduce search space. */ + if (count <= pool->npages/2) { + i = 0; + list_for_each(p, &pool->list) { + if (++i == count) + break; + } + } else { + i = pool->npages + 1; + list_for_each_prev(p, &pool->list) { + if (--i == count) + break; + } + } + /* Cut 'count' number of pages from the pool */ + list_cut_position(pages, &pool->list, p); + pool->npages -= count; + count = 0; +out: + spin_unlock_irqrestore(&pool->lock, irq_flags); + + /* clear the pages coming from the pool if requested */ + if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) { + struct page *page; + + list_for_each_entry(page, pages, lru) { + if (PageHighMem(page)) + clear_highpage(page); + else + clear_page(page_address(page)); + } + } + + /* If pool didn't have enough pages allocate new one. */ + if (count) { + gfp_t gfp_flags = pool->gfp_flags; + + /* set zero flag for page allocation if required */ + if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (ttm_flags & TTM_PAGE_FLAG_NO_RETRY) + gfp_flags |= __GFP_RETRY_MAYFAIL; + + /* ttm_alloc_new_pages doesn't reference pool so we can run + * multiple requests in parallel. + **/ + r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate, + count, order); + } + + return r; +} + +/* Put all pages in pages list to correct pool to wait for reuse */ +static void ttm_put_pages(struct page **pages, unsigned npages, int flags, + enum ttm_caching_state cstate) +{ + struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); +#endif + unsigned long irq_flags; + unsigned i; + + if (pool == NULL) { + /* No pool for this memory type so free the pages */ + i = 0; + while (i < npages) { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct page *p = pages[i]; +#endif + unsigned order = 0, j; + + if (!pages[i]) { + ++i; + continue; + } + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (!(flags & TTM_PAGE_FLAG_DMA32) && + (npages - i) >= HPAGE_PMD_NR) { + for (j = 1; j < HPAGE_PMD_NR; ++j) + if (++p != pages[i + j]) + break; + + if (j == HPAGE_PMD_NR) + order = HPAGE_PMD_ORDER; + } +#endif + + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); + __free_pages(pages[i], order); + + j = 1 << order; + while (j) { + pages[i++] = NULL; + --j; + } + } + return; + } + + i = 0; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (huge) { + unsigned max_size, n2free; + + spin_lock_irqsave(&huge->lock, irq_flags); + while ((npages - i) >= HPAGE_PMD_NR) { + struct page *p = pages[i]; + unsigned j; + + if (!p) + break; + + for (j = 1; j < HPAGE_PMD_NR; ++j) + if (++p != pages[i + j]) + break; + + if (j != HPAGE_PMD_NR) + break; + + list_add_tail(&pages[i]->lru, &huge->list); + + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[i++] = NULL; + huge->npages++; + } + + /* Check that we don't go over the pool limit */ + max_size = _manager->options.max_size; + max_size /= HPAGE_PMD_NR; + if (huge->npages > max_size) + n2free = huge->npages - max_size; + else + n2free = 0; + spin_unlock_irqrestore(&huge->lock, irq_flags); + if (n2free) + ttm_page_pool_free(huge, n2free, false); + } +#endif + + spin_lock_irqsave(&pool->lock, irq_flags); + while (i < npages) { + if (pages[i]) { + if (page_count(pages[i]) != 1) + pr_err("Erroneous page count. Leaking pages.\n"); + list_add_tail(&pages[i]->lru, &pool->list); + pages[i] = NULL; + pool->npages++; + } + ++i; + } + /* Check that we don't go over the pool limit */ + npages = 0; + if (pool->npages > _manager->options.max_size) { + npages = pool->npages - _manager->options.max_size; + /* free at least NUM_PAGES_TO_ALLOC number of pages + * to reduce calls to set_memory_wb */ + if (npages < NUM_PAGES_TO_ALLOC) + npages = NUM_PAGES_TO_ALLOC; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + if (npages) + ttm_page_pool_free(pool, npages, false); +} + +/* + * On success pages list will hold count number of correctly + * cached pages. + */ +static int ttm_get_pages(struct page **pages, unsigned npages, int flags, + enum ttm_caching_state cstate) +{ + struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); +#endif + struct list_head plist; + struct page *p = NULL; + unsigned count, first; + int r; + + /* No pool for cached pages */ + if (pool == NULL) { + gfp_t gfp_flags = GFP_USER; + unsigned i; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + unsigned j; +#endif + + /* set zero flag for page allocation if required */ + if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (flags & TTM_PAGE_FLAG_NO_RETRY) + gfp_flags |= __GFP_RETRY_MAYFAIL; + + if (flags & TTM_PAGE_FLAG_DMA32) + gfp_flags |= GFP_DMA32; + else + gfp_flags |= GFP_HIGHUSER; + + i = 0; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (!(gfp_flags & GFP_DMA32)) { + while (npages >= HPAGE_PMD_NR) { + gfp_t huge_flags = gfp_flags; + + huge_flags |= GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | + __GFP_KSWAPD_RECLAIM; + huge_flags &= ~__GFP_MOVABLE; + huge_flags &= ~__GFP_COMP; + p = alloc_pages(huge_flags, HPAGE_PMD_ORDER); + if (!p) + break; + + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[i++] = p++; + + npages -= HPAGE_PMD_NR; + } + } +#endif + + first = i; + while (npages) { + p = alloc_page(gfp_flags); + if (!p) { + pr_debug("Unable to allocate page\n"); + return -ENOMEM; + } + + /* Swap the pages if we detect consecutive order */ + if (i > first && pages[i - 1] == p - 1) + swap(p, pages[i - 1]); + + pages[i++] = p; + --npages; + } + return 0; + } + + count = 0; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (huge && npages >= HPAGE_PMD_NR) { + INIT_LIST_HEAD(&plist); + ttm_page_pool_get_pages(huge, &plist, flags, cstate, + npages / HPAGE_PMD_NR, + HPAGE_PMD_ORDER); + + list_for_each_entry(p, &plist, lru) { + unsigned j; + + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[count++] = &p[j]; + } + } +#endif + + INIT_LIST_HEAD(&plist); + r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, + npages - count, 0); + + first = count; + list_for_each_entry(p, &plist, lru) { + struct page *tmp = p; + + /* Swap the pages if we detect consecutive order */ + if (count > first && pages[count - 1] == tmp - 1) + swap(tmp, pages[count - 1]); + pages[count++] = tmp; + } + + if (r) { + /* If there is any pages in the list put them back to + * the pool. + */ + pr_debug("Failed to allocate extra pages for large request\n"); + ttm_put_pages(pages, count, flags, cstate); + return r; + } + + return 0; +} + +static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, + char *name, unsigned int order) +{ + spin_lock_init(&pool->lock); + pool->fill_lock = false; + INIT_LIST_HEAD(&pool->list); + pool->npages = pool->nfrees = 0; + pool->gfp_flags = flags; + pool->name = name; + pool->order = order; +} + +int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) +{ + int ret; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + unsigned order = HPAGE_PMD_ORDER; +#else + unsigned order = 0; +#endif + + WARN_ON(_manager); + + pr_info("Initializing pool allocator\n"); + + _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); + if (!_manager) + return -ENOMEM; + + ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0); + + ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0); + + ttm_page_pool_init_locked(&_manager->wc_pool_dma32, + GFP_USER | GFP_DMA32, "wc dma", 0); + + ttm_page_pool_init_locked(&_manager->uc_pool_dma32, + GFP_USER | GFP_DMA32, "uc dma", 0); + + ttm_page_pool_init_locked(&_manager->wc_pool_huge, + (GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | + __GFP_KSWAPD_RECLAIM) & + ~(__GFP_MOVABLE | __GFP_COMP), + "wc huge", order); + + ttm_page_pool_init_locked(&_manager->uc_pool_huge, + (GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | + __GFP_KSWAPD_RECLAIM) & + ~(__GFP_MOVABLE | __GFP_COMP) + , "uc huge", order); + + _manager->options.max_size = max_pages; + _manager->options.small = SMALL_ALLOCATION; + _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; + + ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, + &glob->kobj, "pool"); + if (unlikely(ret != 0)) + goto error; + + ret = ttm_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; + return 0; + +error: + kobject_put(&_manager->kobj); + _manager = NULL; + return ret; +} + +void ttm_page_alloc_fini(void) +{ + int i; + + pr_info("Finalizing pool allocator\n"); + ttm_pool_mm_shrink_fini(_manager); + + /* OK to use static buffer since global mutex is no longer used. */ + for (i = 0; i < NUM_POOLS; ++i) + ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true); + + kobject_put(&_manager->kobj); + _manager = NULL; +} + +static void +ttm_pool_unpopulate_helper(struct ttm_tt *ttm, unsigned mem_count_update) +{ + struct ttm_mem_global *mem_glob = &ttm_mem_glob; + unsigned i; + + if (mem_count_update == 0) + goto put_pages; + + for (i = 0; i < mem_count_update; ++i) { + if (!ttm->pages[i]) + continue; + + ttm_mem_global_free_page(mem_glob, ttm->pages[i], PAGE_SIZE); + } + +put_pages: + ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags, + ttm->caching_state); + ttm_tt_set_unpopulated(ttm); +} + +int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) +{ + struct ttm_mem_global *mem_glob = &ttm_mem_glob; + unsigned i; + int ret; + + if (ttm_tt_is_populated(ttm)) + return 0; + + if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx)) + return -ENOMEM; + + ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags, + ttm->caching_state); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate_helper(ttm, 0); + return ret; + } + + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + PAGE_SIZE, ctx); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate_helper(ttm, i); + return -ENOMEM; + } + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return ret; + } + } + + ttm_tt_set_populated(ttm); + return 0; +} +EXPORT_SYMBOL(ttm_pool_populate); + +void ttm_pool_unpopulate(struct ttm_tt *ttm) +{ + ttm_pool_unpopulate_helper(ttm, ttm->num_pages); +} +EXPORT_SYMBOL(ttm_pool_unpopulate); + +int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, + struct ttm_operation_ctx *ctx) +{ + unsigned i, j; + int r; + + r = ttm_pool_populate(&tt->ttm, ctx); + if (r) + return r; + + for (i = 0; i < tt->ttm.num_pages; ++i) { + struct page *p = tt->ttm.pages[i]; + size_t num_pages = 1; + + for (j = i + 1; j < tt->ttm.num_pages; ++j) { + if (++p != tt->ttm.pages[j]) + break; + + ++num_pages; + } + + tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i], + 0, num_pages * PAGE_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, tt->dma_address[i])) { + while (i--) { + dma_unmap_page(dev, tt->dma_address[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); + tt->dma_address[i] = 0; + } + ttm_pool_unpopulate(&tt->ttm); + return -EFAULT; + } + + for (j = 1; j < num_pages; ++j) { + tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE; + ++i; + } + } + return 0; +} +EXPORT_SYMBOL(ttm_populate_and_map_pages); + +void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt) +{ + unsigned i, j; + + for (i = 0; i < tt->ttm.num_pages;) { + struct page *p = tt->ttm.pages[i]; + size_t num_pages = 1; + + if (!tt->dma_address[i] || !tt->ttm.pages[i]) { + ++i; + continue; + } + + for (j = i + 1; j < tt->ttm.num_pages; ++j) { + if (++p != tt->ttm.pages[j]) + break; + + ++num_pages; + } + + dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE, + DMA_BIDIRECTIONAL); + + i += num_pages; + } + ttm_pool_unpopulate(&tt->ttm); +} +EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages); + +int ttm_page_alloc_debugfs(struct seq_file *m, void *data) +{ + struct ttm_page_pool *p; + unsigned i; + char *h[] = {"pool", "refills", "pages freed", "size"}; + if (!_manager) { + seq_printf(m, "No pool allocator running.\n"); + return 0; + } + seq_printf(m, "%7s %12s %13s %8s\n", + h[0], h[1], h[2], h[3]); + for (i = 0; i < NUM_POOLS; ++i) { + p = &_manager->pools[i]; + + seq_printf(m, "%7s %12ld %13ld %8d\n", + p->name, p->nrefills, + p->nfrees, p->npages); + } + return 0; +} +EXPORT_SYMBOL(ttm_page_alloc_debugfs); diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c new file mode 100644 index 000000000..5e2df1168 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c @@ -0,0 +1,1239 @@ +/* + * Copyright 2011 (c) Oracle Corp. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Author: Konrad Rzeszutek Wilk + */ + +/* + * A simple DMA pool losely based on dmapool.c. It has certain advantages + * over the DMA pools: + * - Pool collects resently freed pages for reuse (and hooks up to + * the shrinker). + * - Tracks currently in use pages + * - Tracks whether the page is UC, WB or cached (and reverts to WB + * when freed). + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include /* for seq_printf */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) +#define SMALL_ALLOCATION 4 +#define FREE_ALL_PAGES (~0U) +#define VADDR_FLAG_HUGE_POOL 1UL +#define VADDR_FLAG_UPDATED_COUNT 2UL + +enum pool_type { + IS_UNDEFINED = 0, + IS_WC = 1 << 1, + IS_UC = 1 << 2, + IS_CACHED = 1 << 3, + IS_DMA32 = 1 << 4, + IS_HUGE = 1 << 5 +}; + +/* + * The pool structure. There are up to nine pools: + * - generic (not restricted to DMA32): + * - write combined, uncached, cached. + * - dma32 (up to 2^32 - so up 4GB): + * - write combined, uncached, cached. + * - huge (not restricted to DMA32): + * - write combined, uncached, cached. + * for each 'struct device'. The 'cached' is for pages that are actively used. + * The other ones can be shrunk by the shrinker API if neccessary. + * @pools: The 'struct device->dma_pools' link. + * @type: Type of the pool + * @lock: Protects the free_list from concurrnet access. Must be + * used with irqsave/irqrestore variants because pool allocator maybe called + * from delayed work. + * @free_list: Pool of pages that are free to be used. No order requirements. + * @dev: The device that is associated with these pools. + * @size: Size used during DMA allocation. + * @npages_free: Count of available pages for re-use. + * @npages_in_use: Count of pages that are in use. + * @nfrees: Stats when pool is shrinking. + * @nrefills: Stats when the pool is grown. + * @gfp_flags: Flags to pass for alloc_page. + * @name: Name of the pool. + * @dev_name: Name derieved from dev - similar to how dev_info works. + * Used during shutdown as the dev_info during release is unavailable. + */ +struct dma_pool { + struct list_head pools; /* The 'struct device->dma_pools link */ + enum pool_type type; + spinlock_t lock; + struct list_head free_list; + struct device *dev; + unsigned size; + unsigned npages_free; + unsigned npages_in_use; + unsigned long nfrees; /* Stats when shrunk. */ + unsigned long nrefills; /* Stats when grown. */ + gfp_t gfp_flags; + char name[13]; /* "cached dma32" */ + char dev_name[64]; /* Constructed from dev */ +}; + +/* + * The accounting page keeping track of the allocated page along with + * the DMA address. + * @page_list: The link to the 'page_list' in 'struct dma_pool'. + * @vaddr: The virtual address of the page and a flag if the page belongs to a + * huge pool + * @dma: The bus address of the page. If the page is not allocated + * via the DMA API, it will be -1. + */ +struct dma_page { + struct list_head page_list; + unsigned long vaddr; + struct page *p; + dma_addr_t dma; +}; + +/* + * Limits for the pool. They are handled without locks because only place where + * they may change is in sysfs store. They won't have immediate effect anyway + * so forcing serialization to access them is pointless. + */ + +struct ttm_pool_opts { + unsigned alloc_size; + unsigned max_size; + unsigned small; +}; + +/* + * Contains the list of all of the 'struct device' and their corresponding + * DMA pools. Guarded by _mutex->lock. + * @pools: The link to 'struct ttm_pool_manager->pools' + * @dev: The 'struct device' associated with the 'pool' + * @pool: The 'struct dma_pool' associated with the 'dev' + */ +struct device_pools { + struct list_head pools; + struct device *dev; + struct dma_pool *pool; +}; + +/* + * struct ttm_pool_manager - Holds memory pools for fast allocation + * + * @lock: Lock used when adding/removing from pools + * @pools: List of 'struct device' and 'struct dma_pool' tuples. + * @options: Limits for the pool. + * @npools: Total amount of pools in existence. + * @shrinker: The structure used by [un|]register_shrinker + */ +struct ttm_pool_manager { + struct mutex lock; + struct list_head pools; + struct ttm_pool_opts options; + unsigned npools; + struct shrinker mm_shrink; + struct kobject kobj; +}; + +static struct ttm_pool_manager *_manager; + +static struct attribute ttm_page_pool_max = { + .name = "pool_max_size", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_small = { + .name = "pool_small_allocation", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_alloc_size = { + .name = "pool_allocation_size", + .mode = S_IRUGO | S_IWUSR +}; + +static struct attribute *ttm_pool_attrs[] = { + &ttm_page_pool_max, + &ttm_page_pool_small, + &ttm_page_pool_alloc_size, + NULL +}; + +static void ttm_pool_kobj_release(struct kobject *kobj) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + kfree(m); +} + +static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, + const char *buffer, size_t size) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + int chars; + unsigned val; + + chars = sscanf(buffer, "%u", &val); + if (chars == 0) + return size; + + /* Convert kb to number of pages */ + val = val / (PAGE_SIZE >> 10); + + if (attr == &ttm_page_pool_max) { + m->options.max_size = val; + } else if (attr == &ttm_page_pool_small) { + m->options.small = val; + } else if (attr == &ttm_page_pool_alloc_size) { + if (val > NUM_PAGES_TO_ALLOC*8) { + pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + return size; + } else if (val > NUM_PAGES_TO_ALLOC) { + pr_warn("Setting allocation size to larger than %lu is not recommended\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + } + m->options.alloc_size = val; + } + + return size; +} + +static ssize_t ttm_pool_show(struct kobject *kobj, struct attribute *attr, + char *buffer) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + unsigned val = 0; + + if (attr == &ttm_page_pool_max) + val = m->options.max_size; + else if (attr == &ttm_page_pool_small) + val = m->options.small; + else if (attr == &ttm_page_pool_alloc_size) + val = m->options.alloc_size; + + val = val * (PAGE_SIZE >> 10); + + return snprintf(buffer, PAGE_SIZE, "%u\n", val); +} + +static const struct sysfs_ops ttm_pool_sysfs_ops = { + .show = &ttm_pool_show, + .store = &ttm_pool_store, +}; + +static struct kobj_type ttm_pool_kobj_type = { + .release = &ttm_pool_kobj_release, + .sysfs_ops = &ttm_pool_sysfs_ops, + .default_attrs = ttm_pool_attrs, +}; + +static int ttm_set_pages_caching(struct dma_pool *pool, + struct page **pages, unsigned cpages) +{ + int r = 0; + /* Set page caching */ + if (pool->type & IS_UC) { + r = ttm_set_pages_array_uc(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to uc!\n", + pool->dev_name, cpages); + } + if (pool->type & IS_WC) { + r = ttm_set_pages_array_wc(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to wc!\n", + pool->dev_name, cpages); + } + return r; +} + +static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) +{ + unsigned long attrs = 0; + dma_addr_t dma = d_page->dma; + d_page->vaddr &= ~VADDR_FLAG_HUGE_POOL; + if (pool->type & IS_HUGE) + attrs = DMA_ATTR_NO_WARN; + + dma_free_attrs(pool->dev, pool->size, (void *)d_page->vaddr, dma, attrs); + + kfree(d_page); + d_page = NULL; +} +static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) +{ + struct dma_page *d_page; + unsigned long attrs = 0; + void *vaddr; + + d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); + if (!d_page) + return NULL; + + if (pool->type & IS_HUGE) + attrs = DMA_ATTR_NO_WARN; + + vaddr = dma_alloc_attrs(pool->dev, pool->size, &d_page->dma, + pool->gfp_flags, attrs); + if (vaddr) { + if (is_vmalloc_addr(vaddr)) + d_page->p = vmalloc_to_page(vaddr); + else + d_page->p = virt_to_page(vaddr); + d_page->vaddr = (unsigned long)vaddr; + if (pool->type & IS_HUGE) + d_page->vaddr |= VADDR_FLAG_HUGE_POOL; + } else { + kfree(d_page); + d_page = NULL; + } + return d_page; +} +static enum pool_type ttm_to_type(int flags, enum ttm_caching_state cstate) +{ + enum pool_type type = IS_UNDEFINED; + + if (flags & TTM_PAGE_FLAG_DMA32) + type |= IS_DMA32; + if (cstate == tt_cached) + type |= IS_CACHED; + else if (cstate == tt_uncached) + type |= IS_UC; + else + type |= IS_WC; + + return type; +} + +static void ttm_pool_update_free_locked(struct dma_pool *pool, + unsigned freed_pages) +{ + pool->npages_free -= freed_pages; + pool->nfrees += freed_pages; + +} + +/* set memory back to wb and free the pages. */ +static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) +{ + struct page *page = d_page->p; + unsigned num_pages; + + /* Don't set WB on WB page pool. */ + if (!(pool->type & IS_CACHED)) { + num_pages = pool->size / PAGE_SIZE; + if (ttm_set_pages_wb(page, num_pages)) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, num_pages); + } + + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); +} + +static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, + struct page *pages[], unsigned npages) +{ + struct dma_page *d_page, *tmp; + + if (pool->type & IS_HUGE) { + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) + ttm_dma_page_put(pool, d_page); + + return; + } + + /* Don't set WB on WB page pool. */ + if (npages && !(pool->type & IS_CACHED) && + ttm_set_pages_array_wb(pages, npages)) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, npages); + + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); + } +} + +/* + * Free pages from pool. + * + * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC + * number of pages in one go. + * + * @pool: to free the pages from + * @nr_free: If set to true will free all pages in pool + * @use_static: Safe to use static buffer + **/ +static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free, + bool use_static) +{ + static struct page *static_buf[NUM_PAGES_TO_ALLOC]; + unsigned long irq_flags; + struct dma_page *dma_p, *tmp; + struct page **pages_to_free; + struct list_head d_pages; + unsigned freed_pages = 0, + npages_to_free = nr_free; + + if (NUM_PAGES_TO_ALLOC < nr_free) + npages_to_free = NUM_PAGES_TO_ALLOC; + + if (use_static) + pages_to_free = static_buf; + else + pages_to_free = kmalloc_array(npages_to_free, + sizeof(struct page *), + GFP_KERNEL); + + if (!pages_to_free) { + pr_debug("%s: Failed to allocate memory for pool free operation\n", + pool->dev_name); + return 0; + } + INIT_LIST_HEAD(&d_pages); +restart: + spin_lock_irqsave(&pool->lock, irq_flags); + + /* We picking the oldest ones off the list */ + list_for_each_entry_safe_reverse(dma_p, tmp, &pool->free_list, + page_list) { + if (freed_pages >= npages_to_free) + break; + + /* Move the dma_page from one list to another. */ + list_move(&dma_p->page_list, &d_pages); + + pages_to_free[freed_pages++] = dma_p->p; + /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ + if (freed_pages >= NUM_PAGES_TO_ALLOC) { + + ttm_pool_update_free_locked(pool, freed_pages); + /** + * Because changing page caching is costly + * we unlock the pool to prevent stalling. + */ + spin_unlock_irqrestore(&pool->lock, irq_flags); + + ttm_dma_pages_put(pool, &d_pages, pages_to_free, + freed_pages); + + INIT_LIST_HEAD(&d_pages); + + if (likely(nr_free != FREE_ALL_PAGES)) + nr_free -= freed_pages; + + if (NUM_PAGES_TO_ALLOC >= nr_free) + npages_to_free = nr_free; + else + npages_to_free = NUM_PAGES_TO_ALLOC; + + freed_pages = 0; + + /* free all so restart the processing */ + if (nr_free) + goto restart; + + /* Not allowed to fall through or break because + * following context is inside spinlock while we are + * outside here. + */ + goto out; + + } + } + + /* remove range of pages from the pool */ + if (freed_pages) { + ttm_pool_update_free_locked(pool, freed_pages); + nr_free -= freed_pages; + } + + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (freed_pages) + ttm_dma_pages_put(pool, &d_pages, pages_to_free, freed_pages); +out: + if (pages_to_free != static_buf) + kfree(pages_to_free); + return nr_free; +} + +static void ttm_dma_free_pool(struct device *dev, enum pool_type type) +{ + struct device_pools *p; + struct dma_pool *pool; + + if (!dev) + return; + + mutex_lock(&_manager->lock); + list_for_each_entry_reverse(p, &_manager->pools, pools) { + if (p->dev != dev) + continue; + pool = p->pool; + if (pool->type != type) + continue; + + list_del(&p->pools); + kfree(p); + _manager->npools--; + break; + } + list_for_each_entry_reverse(pool, &dev->dma_pools, pools) { + if (pool->type != type) + continue; + /* Takes a spinlock.. */ + /* OK to use static buffer since global mutex is held. */ + ttm_dma_page_pool_free(pool, FREE_ALL_PAGES, true); + WARN_ON(((pool->npages_in_use + pool->npages_free) != 0)); + /* This code path is called after _all_ references to the + * struct device has been dropped - so nobody should be + * touching it. In case somebody is trying to _add_ we are + * guarded by the mutex. */ + list_del(&pool->pools); + kfree(pool); + break; + } + mutex_unlock(&_manager->lock); +} + +/* + * On free-ing of the 'struct device' this deconstructor is run. + * Albeit the pool might have already been freed earlier. + */ +static void ttm_dma_pool_release(struct device *dev, void *res) +{ + struct dma_pool *pool = *(struct dma_pool **)res; + + if (pool) + ttm_dma_free_pool(dev, pool->type); +} + +static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data) +{ + return *(struct dma_pool **)res == match_data; +} + +static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, + enum pool_type type) +{ + const char *n[] = {"wc", "uc", "cached", " dma32", "huge"}; + enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_HUGE}; + struct device_pools *sec_pool = NULL; + struct dma_pool *pool = NULL, **ptr; + unsigned i; + int ret = -ENODEV; + char *p; + + if (!dev) + return NULL; + + ptr = devres_alloc(ttm_dma_pool_release, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return NULL; + + ret = -ENOMEM; + + pool = kmalloc_node(sizeof(struct dma_pool), GFP_KERNEL, + dev_to_node(dev)); + if (!pool) + goto err_mem; + + sec_pool = kmalloc_node(sizeof(struct device_pools), GFP_KERNEL, + dev_to_node(dev)); + if (!sec_pool) + goto err_mem; + + INIT_LIST_HEAD(&sec_pool->pools); + sec_pool->dev = dev; + sec_pool->pool = pool; + + INIT_LIST_HEAD(&pool->free_list); + INIT_LIST_HEAD(&pool->pools); + spin_lock_init(&pool->lock); + pool->dev = dev; + pool->npages_free = pool->npages_in_use = 0; + pool->nfrees = 0; + pool->gfp_flags = flags; + if (type & IS_HUGE) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + pool->size = HPAGE_PMD_SIZE; +#else + BUG(); +#endif + else + pool->size = PAGE_SIZE; + pool->type = type; + pool->nrefills = 0; + p = pool->name; + for (i = 0; i < ARRAY_SIZE(t); i++) { + if (type & t[i]) { + p += scnprintf(p, sizeof(pool->name) - (p - pool->name), + "%s", n[i]); + } + } + *p = 0; + /* We copy the name for pr_ calls b/c when dma_pool_destroy is called + * - the kobj->name has already been deallocated.*/ + snprintf(pool->dev_name, sizeof(pool->dev_name), "%s %s", + dev_driver_string(dev), dev_name(dev)); + mutex_lock(&_manager->lock); + /* You can get the dma_pool from either the global: */ + list_add(&sec_pool->pools, &_manager->pools); + _manager->npools++; + /* or from 'struct device': */ + list_add(&pool->pools, &dev->dma_pools); + mutex_unlock(&_manager->lock); + + *ptr = pool; + devres_add(dev, ptr); + + return pool; +err_mem: + devres_free(ptr); + kfree(sec_pool); + kfree(pool); + return ERR_PTR(ret); +} + +static struct dma_pool *ttm_dma_find_pool(struct device *dev, + enum pool_type type) +{ + struct dma_pool *pool, *tmp; + + if (type == IS_UNDEFINED) + return NULL; + + /* NB: We iterate on the 'struct dev' which has no spinlock, but + * it does have a kref which we have taken. The kref is taken during + * graphic driver loading - in the drm_pci_init it calls either + * pci_dev_get or pci_register_driver which both end up taking a kref + * on 'struct device'. + * + * On teardown, the graphic drivers end up quiescing the TTM (put_pages) + * and calls the dev_res deconstructors: ttm_dma_pool_release. The nice + * thing is at that point of time there are no pages associated with the + * driver so this function will not be called. + */ + list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) + if (pool->type == type) + return pool; + return NULL; +} + +/* + * Free pages the pages that failed to change the caching state. If there + * are pages that have changed their caching state already put them to the + * pool. + */ +static void ttm_dma_handle_caching_state_failure(struct dma_pool *pool, + struct list_head *d_pages, + struct page **failed_pages, + unsigned cpages) +{ + struct dma_page *d_page, *tmp; + struct page *p; + unsigned i = 0; + + p = failed_pages[0]; + if (!p) + return; + /* Find the failed page. */ + list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { + if (d_page->p != p) + continue; + /* .. and then progress over the full list. */ + list_del(&d_page->page_list); + __ttm_dma_free_page(pool, d_page); + if (++i < cpages) + p = failed_pages[i]; + else + break; + } + +} + +/* + * Allocate 'count' pages, and put 'need' number of them on the + * 'pages' and as well on the 'dma_address' starting at 'dma_offset' offset. + * The full list of pages should also be on 'd_pages'. + * We return zero for success, and negative numbers as errors. + */ +static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, + struct list_head *d_pages, + unsigned count) +{ + struct page **caching_array; + struct dma_page *dma_p; + struct page *p; + int r = 0; + unsigned i, j, npages, cpages; + unsigned max_cpages = min(count, + (unsigned)(PAGE_SIZE/sizeof(struct page *))); + + /* allocate array for page caching change */ + caching_array = kmalloc_array(max_cpages, sizeof(struct page *), + GFP_KERNEL); + + if (!caching_array) { + pr_debug("%s: Unable to allocate table for new pages\n", + pool->dev_name); + return -ENOMEM; + } + + if (count > 1) + pr_debug("%s: (%s:%d) Getting %d pages\n", + pool->dev_name, pool->name, current->pid, count); + + for (i = 0, cpages = 0; i < count; ++i) { + dma_p = __ttm_dma_alloc_page(pool); + if (!dma_p) { + pr_debug("%s: Unable to get page %u\n", + pool->dev_name, i); + + /* store already allocated pages in the pool after + * setting the caching state */ + if (cpages) { + r = ttm_set_pages_caching(pool, caching_array, + cpages); + if (r) + ttm_dma_handle_caching_state_failure( + pool, d_pages, caching_array, + cpages); + } + r = -ENOMEM; + goto out; + } + p = dma_p->p; + list_add(&dma_p->page_list, d_pages); + +#ifdef CONFIG_HIGHMEM + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ + if (PageHighMem(p)) + continue; +#endif + + npages = pool->size / PAGE_SIZE; + for (j = 0; j < npages; ++j) { + caching_array[cpages++] = p + j; + if (cpages == max_cpages) { + /* Note: Cannot hold the spinlock */ + r = ttm_set_pages_caching(pool, caching_array, + cpages); + if (r) { + ttm_dma_handle_caching_state_failure( + pool, d_pages, caching_array, + cpages); + goto out; + } + cpages = 0; + } + } + } + + if (cpages) { + r = ttm_set_pages_caching(pool, caching_array, cpages); + if (r) + ttm_dma_handle_caching_state_failure(pool, d_pages, + caching_array, cpages); + } +out: + kfree(caching_array); + return r; +} + +/* + * @return count of pages still required to fulfill the request. + */ +static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, + unsigned long *irq_flags) +{ + unsigned count = _manager->options.small; + int r = pool->npages_free; + + if (count > pool->npages_free) { + struct list_head d_pages; + + INIT_LIST_HEAD(&d_pages); + + spin_unlock_irqrestore(&pool->lock, *irq_flags); + + /* Returns how many more are neccessary to fulfill the + * request. */ + r = ttm_dma_pool_alloc_new_pages(pool, &d_pages, count); + + spin_lock_irqsave(&pool->lock, *irq_flags); + if (!r) { + /* Add the fresh to the end.. */ + list_splice(&d_pages, &pool->free_list); + ++pool->nrefills; + pool->npages_free += count; + r = count; + } else { + struct dma_page *d_page; + unsigned cpages = 0; + + pr_debug("%s: Failed to fill %s pool (r:%d)!\n", + pool->dev_name, pool->name, r); + + list_for_each_entry(d_page, &d_pages, page_list) { + cpages++; + } + list_splice_tail(&d_pages, &pool->free_list); + pool->npages_free += cpages; + r = cpages; + } + } + return r; +} + +/* + * The populate list is actually a stack (not that is matters as TTM + * allocates one page at a time. + * return dma_page pointer if success, otherwise NULL. + */ +static struct dma_page *ttm_dma_pool_get_pages(struct dma_pool *pool, + struct ttm_dma_tt *ttm_dma, + unsigned index) +{ + struct dma_page *d_page = NULL; + struct ttm_tt *ttm = &ttm_dma->ttm; + unsigned long irq_flags; + int count; + + spin_lock_irqsave(&pool->lock, irq_flags); + count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); + if (count) { + d_page = list_first_entry(&pool->free_list, struct dma_page, page_list); + ttm->pages[index] = d_page->p; + ttm_dma->dma_address[index] = d_page->dma; + list_move_tail(&d_page->page_list, &ttm_dma->pages_list); + pool->npages_in_use += 1; + pool->npages_free -= 1; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + return d_page; +} + +static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + gfp_t gfp_flags; + + if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) + gfp_flags = GFP_USER | GFP_DMA32; + else + gfp_flags = GFP_HIGHUSER; + if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (huge) { + gfp_flags |= GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | + __GFP_KSWAPD_RECLAIM; + gfp_flags &= ~__GFP_MOVABLE; + gfp_flags &= ~__GFP_COMP; + } + + if (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY) + gfp_flags |= __GFP_RETRY_MAYFAIL; + + return gfp_flags; +} + +/* + * On success pages list will hold count number of correctly + * cached pages. On failure will hold the negative return value (-ENOMEM, etc). + */ +int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev, + struct ttm_operation_ctx *ctx) +{ + struct ttm_mem_global *mem_glob = &ttm_mem_glob; + struct ttm_tt *ttm = &ttm_dma->ttm; + unsigned long num_pages = ttm->num_pages; + struct dma_pool *pool; + struct dma_page *d_page; + enum pool_type type; + unsigned i; + int ret; + + if (ttm_tt_is_populated(ttm)) + return 0; + + if (ttm_check_under_lowerlimit(mem_glob, num_pages, ctx)) + return -ENOMEM; + + INIT_LIST_HEAD(&ttm_dma->pages_list); + i = 0; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) + goto skip_huge; + + pool = ttm_dma_find_pool(dev, type | IS_HUGE); + if (!pool) { + gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, true); + + pool = ttm_dma_pool_init(dev, gfp_flags, type | IS_HUGE); + if (IS_ERR_OR_NULL(pool)) + goto skip_huge; + } + + while (num_pages >= HPAGE_PMD_NR) { + unsigned j; + + d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (!d_page) + break; + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + pool->size, ctx); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + + d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; + for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) { + ttm->pages[j] = ttm->pages[j - 1] + 1; + ttm_dma->dma_address[j] = ttm_dma->dma_address[j - 1] + + PAGE_SIZE; + } + + i += HPAGE_PMD_NR; + num_pages -= HPAGE_PMD_NR; + } + +skip_huge: +#endif + + pool = ttm_dma_find_pool(dev, type); + if (!pool) { + gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm_dma, false); + + pool = ttm_dma_pool_init(dev, gfp_flags, type); + if (IS_ERR_OR_NULL(pool)) + return -ENOMEM; + } + + while (num_pages) { + d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i); + if (!d_page) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + pool->size, ctx); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return -ENOMEM; + } + + d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; + ++i; + --num_pages; + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_dma_unpopulate(ttm_dma, dev); + return ret; + } + } + + ttm_tt_set_populated(ttm); + return 0; +} +EXPORT_SYMBOL_GPL(ttm_dma_populate); + +/* Put all pages in pages list to correct pool to wait for reuse */ +void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev) +{ + struct ttm_mem_global *mem_glob = &ttm_mem_glob; + struct ttm_tt *ttm = &ttm_dma->ttm; + struct dma_pool *pool; + struct dma_page *d_page, *next; + enum pool_type type; + bool is_cached = false; + unsigned count, i, npages = 0; + unsigned long irq_flags; + + type = ttm_to_type(ttm->page_flags, ttm->caching_state); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + pool = ttm_dma_find_pool(dev, type | IS_HUGE); + if (pool) { + count = 0; + list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, + page_list) { + if (!(d_page->vaddr & VADDR_FLAG_HUGE_POOL)) + continue; + + count++; + if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { + ttm_mem_global_free_page(mem_glob, d_page->p, + pool->size); + d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; + } + ttm_dma_page_put(pool, d_page); + } + + spin_lock_irqsave(&pool->lock, irq_flags); + pool->npages_in_use -= count; + pool->nfrees += count; + spin_unlock_irqrestore(&pool->lock, irq_flags); + } +#endif + + pool = ttm_dma_find_pool(dev, type); + if (!pool) + return; + + is_cached = (ttm_dma_find_pool(pool->dev, + ttm_to_type(ttm->page_flags, tt_cached)) == pool); + + /* make sure pages array match list and count number of pages */ + count = 0; + list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, + page_list) { + ttm->pages[count] = d_page->p; + count++; + + if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { + ttm_mem_global_free_page(mem_glob, d_page->p, + pool->size); + d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; + } + + if (is_cached) + ttm_dma_page_put(pool, d_page); + } + + spin_lock_irqsave(&pool->lock, irq_flags); + pool->npages_in_use -= count; + if (is_cached) { + pool->nfrees += count; + } else { + pool->npages_free += count; + list_splice(&ttm_dma->pages_list, &pool->free_list); + /* + * Wait to have at at least NUM_PAGES_TO_ALLOC number of pages + * to free in order to minimize calls to set_memory_wb(). + */ + if (pool->npages_free >= (_manager->options.max_size + + NUM_PAGES_TO_ALLOC)) + npages = pool->npages_free - _manager->options.max_size; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + + INIT_LIST_HEAD(&ttm_dma->pages_list); + for (i = 0; i < ttm->num_pages; i++) { + ttm->pages[i] = NULL; + ttm_dma->dma_address[i] = 0; + } + + /* shrink pool if necessary (only on !is_cached pools)*/ + if (npages) + ttm_dma_page_pool_free(pool, npages, false); + ttm_tt_set_unpopulated(ttm); +} +EXPORT_SYMBOL_GPL(ttm_dma_unpopulate); + +/** + * Callback for mm to request pool to reduce number of page held. + * + * XXX: (dchinner) Deadlock warning! + * + * I'm getting sadder as I hear more pathetical whimpers about needing per-pool + * shrinkers + */ +static unsigned long +ttm_dma_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + static unsigned start_pool; + unsigned idx = 0; + unsigned pool_offset; + unsigned shrink_pages = sc->nr_to_scan; + struct device_pools *p; + unsigned long freed = 0; + + if (list_empty(&_manager->pools)) + return SHRINK_STOP; + + if (!mutex_trylock(&_manager->lock)) + return SHRINK_STOP; + if (!_manager->npools) + goto out; + pool_offset = ++start_pool % _manager->npools; + list_for_each_entry(p, &_manager->pools, pools) { + unsigned nr_free; + + if (!p->dev) + continue; + if (shrink_pages == 0) + break; + /* Do it in round-robin fashion. */ + if (++idx < pool_offset) + continue; + nr_free = shrink_pages; + /* OK to use static buffer since global mutex is held. */ + shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free, true); + freed += nr_free - shrink_pages; + + pr_debug("%s: (%s:%d) Asked to shrink %d, have %d more to go\n", + p->pool->dev_name, p->pool->name, current->pid, + nr_free, shrink_pages); + } +out: + mutex_unlock(&_manager->lock); + return freed; +} + +static unsigned long +ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + struct device_pools *p; + unsigned long count = 0; + + if (!mutex_trylock(&_manager->lock)) + return 0; + list_for_each_entry(p, &_manager->pools, pools) + count += p->pool->npages_free; + mutex_unlock(&_manager->lock); + return count; +} + +static int ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) +{ + manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; + manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; + manager->mm_shrink.seeks = 1; + return register_shrinker(&manager->mm_shrink); +} + +static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +{ + unregister_shrinker(&manager->mm_shrink); +} + +int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) +{ + int ret; + + WARN_ON(_manager); + + pr_info("Initializing DMA pool allocator\n"); + + _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); + if (!_manager) + return -ENOMEM; + + mutex_init(&_manager->lock); + INIT_LIST_HEAD(&_manager->pools); + + _manager->options.max_size = max_pages; + _manager->options.small = SMALL_ALLOCATION; + _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; + + /* This takes care of auto-freeing the _manager */ + ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, + &glob->kobj, "dma_pool"); + if (unlikely(ret != 0)) + goto error; + + ret = ttm_dma_pool_mm_shrink_init(_manager); + if (unlikely(ret != 0)) + goto error; + return 0; + +error: + kobject_put(&_manager->kobj); + _manager = NULL; + return ret; +} + +void ttm_dma_page_alloc_fini(void) +{ + struct device_pools *p, *t; + + pr_info("Finalizing DMA pool allocator\n"); + ttm_dma_pool_mm_shrink_fini(_manager); + + list_for_each_entry_safe_reverse(p, t, &_manager->pools, pools) { + dev_dbg(p->dev, "(%s:%d) Freeing.\n", p->pool->name, + current->pid); + WARN_ON(devres_destroy(p->dev, ttm_dma_pool_release, + ttm_dma_pool_match, p->pool)); + ttm_dma_free_pool(p->dev, p->pool->type); + } + kobject_put(&_manager->kobj); + _manager = NULL; +} + +int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) +{ + struct device_pools *p; + struct dma_pool *pool = NULL; + + if (!_manager) { + seq_printf(m, "No pool allocator running.\n"); + return 0; + } + seq_printf(m, " pool refills pages freed inuse available name\n"); + mutex_lock(&_manager->lock); + list_for_each_entry(p, &_manager->pools, pools) { + struct device *dev = p->dev; + if (!dev) + continue; + pool = p->pool; + seq_printf(m, "%13s %12ld %13ld %8d %8d %8s\n", + pool->name, pool->nrefills, + pool->nfrees, pool->npages_in_use, + pool->npages_free, + pool->dev_name); + } + mutex_unlock(&_manager->lock); + return 0; +} +EXPORT_SYMBOL_GPL(ttm_dma_page_alloc_debugfs); diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c new file mode 100644 index 000000000..ce9d127ed --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#include +#include +#include +#include +#include +#include +#include + +/** + * Currently we use a spinlock for the lock, but a mutex *may* be + * more appropriate to reduce scheduling latency if the range manager + * ends up with very fragmented allocation patterns. + */ + +struct ttm_range_manager { + struct ttm_resource_manager manager; + struct drm_mm mm; + spinlock_t lock; +}; + +static inline struct ttm_range_manager *to_range_manager(struct ttm_resource_manager *man) +{ + return container_of(man, struct ttm_range_manager, manager); +} + +static int ttm_range_man_alloc(struct ttm_resource_manager *man, + struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource *mem) +{ + struct ttm_range_manager *rman = to_range_manager(man); + struct drm_mm *mm = &rman->mm; + struct drm_mm_node *node; + enum drm_mm_insert_mode mode; + unsigned long lpfn; + int ret; + + lpfn = place->lpfn; + if (!lpfn) + lpfn = man->size; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + mode = DRM_MM_INSERT_BEST; + if (place->flags & TTM_PL_FLAG_TOPDOWN) + mode = DRM_MM_INSERT_HIGH; + + spin_lock(&rman->lock); + ret = drm_mm_insert_node_in_range(mm, node, + mem->num_pages, + mem->page_alignment, 0, + place->fpfn, lpfn, mode); + spin_unlock(&rman->lock); + + if (unlikely(ret)) { + kfree(node); + } else { + mem->mm_node = node; + mem->start = node->start; + } + + return ret; +} + +static void ttm_range_man_free(struct ttm_resource_manager *man, + struct ttm_resource *mem) +{ + struct ttm_range_manager *rman = to_range_manager(man); + + if (mem->mm_node) { + spin_lock(&rman->lock); + drm_mm_remove_node(mem->mm_node); + spin_unlock(&rman->lock); + + kfree(mem->mm_node); + mem->mm_node = NULL; + } +} + +static const struct ttm_resource_manager_func ttm_range_manager_func; + +int ttm_range_man_init(struct ttm_bo_device *bdev, + unsigned type, bool use_tt, + unsigned long p_size) +{ + struct ttm_resource_manager *man; + struct ttm_range_manager *rman; + + rman = kzalloc(sizeof(*rman), GFP_KERNEL); + if (!rman) + return -ENOMEM; + + man = &rman->manager; + man->use_tt = use_tt; + + man->func = &ttm_range_manager_func; + + ttm_resource_manager_init(man, p_size); + + drm_mm_init(&rman->mm, 0, p_size); + spin_lock_init(&rman->lock); + + ttm_set_driver_manager(bdev, type, &rman->manager); + ttm_resource_manager_set_used(man, true); + return 0; +} +EXPORT_SYMBOL(ttm_range_man_init); + +int ttm_range_man_fini(struct ttm_bo_device *bdev, + unsigned type) +{ + struct ttm_resource_manager *man = ttm_manager_type(bdev, type); + struct ttm_range_manager *rman = to_range_manager(man); + struct drm_mm *mm = &rman->mm; + int ret; + + if (!man) + return 0; + + ttm_resource_manager_set_used(man, false); + + ret = ttm_resource_manager_force_list_clean(bdev, man); + if (ret) + return ret; + + spin_lock(&rman->lock); + drm_mm_clean(mm); + drm_mm_takedown(mm); + spin_unlock(&rman->lock); + + ttm_resource_manager_cleanup(man); + ttm_set_driver_manager(bdev, type, NULL); + kfree(rman); + return 0; +} +EXPORT_SYMBOL(ttm_range_man_fini); + +static void ttm_range_man_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + struct ttm_range_manager *rman = to_range_manager(man); + + spin_lock(&rman->lock); + drm_mm_print(&rman->mm, printer); + spin_unlock(&rman->lock); +} + +static const struct ttm_resource_manager_func ttm_range_manager_func = { + .alloc = ttm_range_man_alloc, + .free = ttm_range_man_free, + .debug = ttm_range_man_debug +}; diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c new file mode 100644 index 000000000..b325b9264 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -0,0 +1,146 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#include +#include + +int ttm_resource_alloc(struct ttm_buffer_object *bo, + const struct ttm_place *place, + struct ttm_resource *res) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, res->mem_type); + + res->mm_node = NULL; + if (!man->func || !man->func->alloc) + return 0; + + return man->func->alloc(man, bo, place, res); +} + +void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource *res) +{ + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, res->mem_type); + + if (man->func && man->func->free) + man->func->free(man, res); + + res->mm_node = NULL; + res->mem_type = TTM_PL_SYSTEM; +} +EXPORT_SYMBOL(ttm_resource_free); + +/** + * ttm_resource_manager_init + * + * @man: memory manager object to init + * @p_size: size managed area in pages. + * + * Initialise core parts of a manager object. + */ +void ttm_resource_manager_init(struct ttm_resource_manager *man, + unsigned long p_size) +{ + unsigned i; + + spin_lock_init(&man->move_lock); + man->size = p_size; + + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + INIT_LIST_HEAD(&man->lru[i]); + man->move = NULL; +} +EXPORT_SYMBOL(ttm_resource_manager_init); + +/* + * ttm_resource_manager_force_list_clean + * + * @bdev - device to use + * @man - manager to use + * + * Force all the objects out of a memory manager until clean. + * Part of memory manager cleanup sequence. + */ +int ttm_resource_manager_force_list_clean(struct ttm_bo_device *bdev, + struct ttm_resource_manager *man) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false, + .flags = TTM_OPT_FLAG_FORCE_ALLOC + }; + struct ttm_bo_global *glob = &ttm_bo_glob; + struct dma_fence *fence; + int ret; + unsigned i; + + /* + * Can't use standard list traversal since we're unlocking. + */ + + spin_lock(&glob->lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + while (!list_empty(&man->lru[i])) { + spin_unlock(&glob->lru_lock); + ret = ttm_mem_evict_first(bdev, man, NULL, &ctx, + NULL); + if (ret) + return ret; + spin_lock(&glob->lru_lock); + } + } + spin_unlock(&glob->lru_lock); + + spin_lock(&man->move_lock); + fence = dma_fence_get(man->move); + spin_unlock(&man->move_lock); + + if (fence) { + ret = dma_fence_wait(fence, false); + dma_fence_put(fence); + if (ret) + return ret; + } + + return 0; +} +EXPORT_SYMBOL(ttm_resource_manager_force_list_clean); + +/** + * ttm_resource_manager_debug + * + * @man: manager type to dump. + * @p: printer to use for debug. + */ +void ttm_resource_manager_debug(struct ttm_resource_manager *man, + struct drm_printer *p) +{ + drm_printf(p, " use_type: %d\n", man->use_type); + drm_printf(p, " use_tt: %d\n", man->use_tt); + drm_printf(p, " size: %llu\n", man->size); + if (man->func && man->func->debug) + (*man->func->debug)(man, p); +} +EXPORT_SYMBOL(ttm_resource_manager_debug); diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c new file mode 100644 index 000000000..f43fa69a1 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -0,0 +1,468 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/************************************************************************** + * + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#define pr_fmt(fmt) "[TTM] " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * Allocates a ttm structure for the given BO. + */ +int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) +{ + struct ttm_bo_device *bdev = bo->bdev; + uint32_t page_flags = 0; + + dma_resv_assert_held(bo->base.resv); + + if (bo->ttm) + return 0; + + if (bdev->need_dma32) + page_flags |= TTM_PAGE_FLAG_DMA32; + + if (bdev->no_retry) + page_flags |= TTM_PAGE_FLAG_NO_RETRY; + + switch (bo->type) { + case ttm_bo_type_device: + if (zero_alloc) + page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; + break; + case ttm_bo_type_kernel: + break; + case ttm_bo_type_sg: + page_flags |= TTM_PAGE_FLAG_SG; + break; + default: + pr_err("Illegal buffer object type\n"); + return -EINVAL; + } + + bo->ttm = bdev->driver->ttm_tt_create(bo, page_flags); + if (unlikely(bo->ttm == NULL)) + return -ENOMEM; + + return 0; +} + +/** + * Allocates storage for pointers to the pages that back the ttm. + */ +static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm) +{ + ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*), + GFP_KERNEL | __GFP_ZERO); + if (!ttm->pages) + return -ENOMEM; + return 0; +} + +static int ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) +{ + ttm->ttm.pages = kvmalloc_array(ttm->ttm.num_pages, + sizeof(*ttm->ttm.pages) + + sizeof(*ttm->dma_address), + GFP_KERNEL | __GFP_ZERO); + if (!ttm->ttm.pages) + return -ENOMEM; + ttm->dma_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages); + return 0; +} + +static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm) +{ + ttm->dma_address = kvmalloc_array(ttm->ttm.num_pages, + sizeof(*ttm->dma_address), + GFP_KERNEL | __GFP_ZERO); + if (!ttm->dma_address) + return -ENOMEM; + return 0; +} + +static int ttm_tt_set_page_caching(struct page *p, + enum ttm_caching_state c_old, + enum ttm_caching_state c_new) +{ + int ret = 0; + + if (PageHighMem(p)) + return 0; + + if (c_old != tt_cached) { + /* p isn't in the default caching state, set it to + * writeback first to free its current memtype. */ + + ret = ttm_set_pages_wb(p, 1); + if (ret) + return ret; + } + + if (c_new == tt_wc) + ret = ttm_set_pages_wc(p, 1); + else if (c_new == tt_uncached) + ret = ttm_set_pages_uc(p, 1); + + return ret; +} + +/* + * Change caching policy for the linear kernel map + * for range of pages in a ttm. + */ + +static int ttm_tt_set_caching(struct ttm_tt *ttm, + enum ttm_caching_state c_state) +{ + int i, j; + struct page *cur_page; + int ret; + + if (ttm->caching_state == c_state) + return 0; + + if (!ttm_tt_is_populated(ttm)) { + /* Change caching but don't populate */ + ttm->caching_state = c_state; + return 0; + } + + if (ttm->caching_state == tt_cached) + drm_clflush_pages(ttm->pages, ttm->num_pages); + + for (i = 0; i < ttm->num_pages; ++i) { + cur_page = ttm->pages[i]; + if (likely(cur_page != NULL)) { + ret = ttm_tt_set_page_caching(cur_page, + ttm->caching_state, + c_state); + if (unlikely(ret != 0)) + goto out_err; + } + } + + ttm->caching_state = c_state; + + return 0; + +out_err: + for (j = 0; j < i; ++j) { + cur_page = ttm->pages[j]; + if (likely(cur_page != NULL)) { + (void)ttm_tt_set_page_caching(cur_page, c_state, + ttm->caching_state); + } + } + + return ret; +} + +int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) +{ + enum ttm_caching_state state; + + if (placement & TTM_PL_FLAG_WC) + state = tt_wc; + else if (placement & TTM_PL_FLAG_UNCACHED) + state = tt_uncached; + else + state = tt_cached; + + return ttm_tt_set_caching(ttm, state); +} +EXPORT_SYMBOL(ttm_tt_set_placement_caching); + +void ttm_tt_destroy_common(struct ttm_bo_device *bdev, struct ttm_tt *ttm) +{ + ttm_tt_unpopulate(bdev, ttm); + + if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && + ttm->swap_storage) + fput(ttm->swap_storage); + + ttm->swap_storage = NULL; +} +EXPORT_SYMBOL(ttm_tt_destroy_common); + +void ttm_tt_destroy(struct ttm_bo_device *bdev, struct ttm_tt *ttm) +{ + bdev->driver->ttm_tt_destroy(bdev, ttm); +} + +static void ttm_tt_init_fields(struct ttm_tt *ttm, + struct ttm_buffer_object *bo, + uint32_t page_flags) +{ + ttm->num_pages = bo->num_pages; + ttm->caching_state = tt_cached; + ttm->page_flags = page_flags; + ttm_tt_set_unpopulated(ttm); + ttm->swap_storage = NULL; + ttm->sg = bo->sg; +} + +int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, + uint32_t page_flags) +{ + ttm_tt_init_fields(ttm, bo, page_flags); + + if (ttm_tt_alloc_page_directory(ttm)) { + pr_err("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} +EXPORT_SYMBOL(ttm_tt_init); + +void ttm_tt_fini(struct ttm_tt *ttm) +{ + kvfree(ttm->pages); + ttm->pages = NULL; +} +EXPORT_SYMBOL(ttm_tt_fini); + +int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo, + uint32_t page_flags) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + + ttm_tt_init_fields(ttm, bo, page_flags); + + INIT_LIST_HEAD(&ttm_dma->pages_list); + if (ttm_dma_tt_alloc_page_directory(ttm_dma)) { + pr_err("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} +EXPORT_SYMBOL(ttm_dma_tt_init); + +int ttm_sg_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_buffer_object *bo, + uint32_t page_flags) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + int ret; + + ttm_tt_init_fields(ttm, bo, page_flags); + + INIT_LIST_HEAD(&ttm_dma->pages_list); + if (page_flags & TTM_PAGE_FLAG_SG) + ret = ttm_sg_tt_alloc_page_directory(ttm_dma); + else + ret = ttm_dma_tt_alloc_page_directory(ttm_dma); + if (ret) { + pr_err("Failed allocating page table\n"); + return -ENOMEM; + } + return 0; +} +EXPORT_SYMBOL(ttm_sg_tt_init); + +void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) +{ + struct ttm_tt *ttm = &ttm_dma->ttm; + + if (ttm->pages) + kvfree(ttm->pages); + else + kvfree(ttm_dma->dma_address); + ttm->pages = NULL; + ttm_dma->dma_address = NULL; +} +EXPORT_SYMBOL(ttm_dma_tt_fini); + +int ttm_tt_swapin(struct ttm_tt *ttm) +{ + struct address_space *swap_space; + struct file *swap_storage; + struct page *from_page; + struct page *to_page; + int i; + int ret = -ENOMEM; + + swap_storage = ttm->swap_storage; + BUG_ON(swap_storage == NULL); + + swap_space = swap_storage->f_mapping; + + for (i = 0; i < ttm->num_pages; ++i) { + gfp_t gfp_mask = mapping_gfp_mask(swap_space); + + gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0); + from_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); + + if (IS_ERR(from_page)) { + ret = PTR_ERR(from_page); + goto out_err; + } + to_page = ttm->pages[i]; + if (unlikely(to_page == NULL)) + goto out_err; + + copy_highpage(to_page, from_page); + put_page(from_page); + } + + if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) + fput(swap_storage); + ttm->swap_storage = NULL; + ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; + + return 0; +out_err: + return ret; +} + +int ttm_tt_swapout(struct ttm_bo_device *bdev, + struct ttm_tt *ttm, struct file *persistent_swap_storage) +{ + struct address_space *swap_space; + struct file *swap_storage; + struct page *from_page; + struct page *to_page; + int i; + int ret = -ENOMEM; + + BUG_ON(ttm->caching_state != tt_cached); + + if (!persistent_swap_storage) { + swap_storage = shmem_file_setup("ttm swap", + ttm->num_pages << PAGE_SHIFT, + 0); + if (IS_ERR(swap_storage)) { + pr_err("Failed allocating swap storage\n"); + return PTR_ERR(swap_storage); + } + } else { + swap_storage = persistent_swap_storage; + } + + swap_space = swap_storage->f_mapping; + + for (i = 0; i < ttm->num_pages; ++i) { + gfp_t gfp_mask = mapping_gfp_mask(swap_space); + + gfp_mask |= (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY ? __GFP_RETRY_MAYFAIL : 0); + + from_page = ttm->pages[i]; + if (unlikely(from_page == NULL)) + continue; + + to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); + if (IS_ERR(to_page)) { + ret = PTR_ERR(to_page); + goto out_err; + } + copy_highpage(to_page, from_page); + set_page_dirty(to_page); + mark_page_accessed(to_page); + put_page(to_page); + } + + ttm_tt_unpopulate(bdev, ttm); + ttm->swap_storage = swap_storage; + ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; + if (persistent_swap_storage) + ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; + + return 0; +out_err: + if (!persistent_swap_storage) + fput(swap_storage); + + return ret; +} + +static void ttm_tt_add_mapping(struct ttm_bo_device *bdev, struct ttm_tt *ttm) +{ + pgoff_t i; + + if (ttm->page_flags & TTM_PAGE_FLAG_SG) + return; + + for (i = 0; i < ttm->num_pages; ++i) + ttm->pages[i]->mapping = bdev->dev_mapping; +} + +int ttm_tt_populate(struct ttm_bo_device *bdev, + struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) +{ + int ret; + + if (!ttm) + return -EINVAL; + + if (ttm_tt_is_populated(ttm)) + return 0; + + if (bdev->driver->ttm_tt_populate) + ret = bdev->driver->ttm_tt_populate(bdev, ttm, ctx); + else + ret = ttm_pool_populate(ttm, ctx); + if (!ret) + ttm_tt_add_mapping(bdev, ttm); + return ret; +} +EXPORT_SYMBOL(ttm_tt_populate); + +static void ttm_tt_clear_mapping(struct ttm_tt *ttm) +{ + pgoff_t i; + struct page **page = ttm->pages; + + if (ttm->page_flags & TTM_PAGE_FLAG_SG) + return; + + for (i = 0; i < ttm->num_pages; ++i) { + (*page)->mapping = NULL; + (*page++)->index = 0; + } +} + +void ttm_tt_unpopulate(struct ttm_bo_device *bdev, + struct ttm_tt *ttm) +{ + if (!ttm_tt_is_populated(ttm)) + return; + + ttm_tt_clear_mapping(ttm); + if (bdev->driver->ttm_tt_unpopulate) + bdev->driver->ttm_tt_unpopulate(bdev, ttm); + else + ttm_pool_unpopulate(ttm); +} -- cgit v1.2.3