diff options
Diffstat (limited to 'drivers/misc/habanalabs/common/device.c')
| -rw-r--r-- | drivers/misc/habanalabs/common/device.c | 2258 |
1 files changed, 2258 insertions, 0 deletions
diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c new file mode 100644 index 000000000..9ee1b6abd --- /dev/null +++ b/drivers/misc/habanalabs/common/device.c @@ -0,0 +1,2258 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright 2016-2022 HabanaLabs, Ltd. + * All Rights Reserved. + */ + +#define pr_fmt(fmt) "habanalabs: " fmt + +#include <uapi/misc/habanalabs.h> +#include "habanalabs.h" + +#include <linux/pci.h> +#include <linux/hwmon.h> + +#include <trace/events/habanalabs.h> + +#define HL_RESET_DELAY_USEC 10000 /* 10ms */ + +enum dma_alloc_type { + DMA_ALLOC_COHERENT, + DMA_ALLOC_CPU_ACCESSIBLE, + DMA_ALLOC_POOL, +}; + +#define MEM_SCRUB_DEFAULT_VAL 0x1122334455667788 + +/* + * hl_set_dram_bar- sets the bar to allow later access to address + * + * @hdev: pointer to habanalabs device structure. + * @addr: the address the caller wants to access. + * @region: the PCI region. + * + * @return: the old BAR base address on success, U64_MAX for failure. + * The caller should set it back to the old address after use. + * + * In case the bar space does not cover the whole address space, + * the bar base address should be set to allow access to a given address. + * This function can be called also if the bar doesn't need to be set, + * in that case it just won't change the base. + */ +static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + u64 bar_base_addr, old_base; + + if (is_power_of_2(prop->dram_pci_bar_size)) + bar_base_addr = addr & ~(prop->dram_pci_bar_size - 0x1ull); + else + bar_base_addr = DIV_ROUND_DOWN_ULL(addr, prop->dram_pci_bar_size) * + prop->dram_pci_bar_size; + + old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr); + + /* in case of success we need to update the new BAR base */ + if (old_base != U64_MAX) + region->region_base = bar_base_addr; + + return old_base; +} + +static int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val, + enum debugfs_access_type acc_type, enum pci_region region_type) +{ + struct pci_mem_region *region = &hdev->pci_mem_region[region_type]; + void __iomem *acc_addr; + u64 old_base = 0, rc; + + if (region_type == PCI_REGION_DRAM) { + old_base = hl_set_dram_bar(hdev, addr, region); + if (old_base == U64_MAX) + return -EIO; + } + + acc_addr = hdev->pcie_bar[region->bar_id] + addr - region->region_base + + region->offset_in_bar; + switch (acc_type) { + case DEBUGFS_READ8: + *val = readb(acc_addr); + break; + case DEBUGFS_WRITE8: + writeb(*val, acc_addr); + break; + case DEBUGFS_READ32: + *val = readl(acc_addr); + break; + case DEBUGFS_WRITE32: + writel(*val, acc_addr); + break; + case DEBUGFS_READ64: + *val = readq(acc_addr); + break; + case DEBUGFS_WRITE64: + writeq(*val, acc_addr); + break; + } + + if (region_type == PCI_REGION_DRAM) { + rc = hl_set_dram_bar(hdev, old_base, region); + if (rc == U64_MAX) + return -EIO; + } + + return 0; +} + +static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, + gfp_t flag, enum dma_alloc_type alloc_type, + const char *caller) +{ + void *ptr = NULL; + + switch (alloc_type) { + case DMA_ALLOC_COHERENT: + ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag); + break; + case DMA_ALLOC_CPU_ACCESSIBLE: + ptr = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); + break; + case DMA_ALLOC_POOL: + ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle); + break; + } + + if (trace_habanalabs_dma_alloc_enabled() && !ZERO_OR_NULL_PTR(ptr)) + trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size, + caller); + + return ptr; +} + +static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, enum dma_alloc_type alloc_type, + const char *caller) +{ + switch (alloc_type) { + case DMA_ALLOC_COHERENT: + hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle); + break; + case DMA_ALLOC_CPU_ACCESSIBLE: + hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, cpu_addr); + break; + case DMA_ALLOC_POOL: + hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle); + break; + } + + trace_habanalabs_dma_free(hdev->dev, (u64) (uintptr_t) cpu_addr, dma_handle, size, caller); +} + +void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, + gfp_t flag, const char *caller) +{ + return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller); +} + +void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, const char *caller) +{ + hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller); +} + +void *hl_cpu_accessible_dma_pool_alloc_caller(struct hl_device *hdev, size_t size, + dma_addr_t *dma_handle, const char *caller) +{ + return hl_dma_alloc_common(hdev, size, dma_handle, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller); +} + +void hl_cpu_accessible_dma_pool_free_caller(struct hl_device *hdev, size_t size, void *vaddr, + const char *caller) +{ + hl_asic_dma_free_common(hdev, size, vaddr, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller); +} + +void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, + dma_addr_t *dma_handle, const char *caller) +{ + return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller); +} + +void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, + const char *caller) +{ + hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller); +} + +int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct scatterlist *sg; + int rc, i; + + rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0); + if (rc) + return rc; + + /* Shift to the device's base physical address of host memory if necessary */ + if (prop->device_dma_offset_for_host_access) + for_each_sgtable_dma_sg(sgt, sg, i) + sg->dma_address += prop->device_dma_offset_for_host_access; + + return 0; +} + +void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + struct scatterlist *sg; + int i; + + /* Cancel the device's base physical address of host memory if necessary */ + if (prop->device_dma_offset_for_host_access) + for_each_sgtable_dma_sg(sgt, sg, i) + sg->dma_address -= prop->device_dma_offset_for_host_access; + + dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0); +} + +/* + * hl_access_cfg_region - access the config region + * + * @hdev: pointer to habanalabs device structure + * @addr: the address to access + * @val: the value to write from or read to + * @acc_type: the type of access (read/write 64/32) + */ +int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, + enum debugfs_access_type acc_type) +{ + struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG]; + u32 val_h, val_l; + + if (!IS_ALIGNED(addr, sizeof(u32))) { + dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32)); + return -EINVAL; + } + + switch (acc_type) { + case DEBUGFS_READ32: + *val = RREG32(addr - cfg_region->region_base); + break; + case DEBUGFS_WRITE32: + WREG32(addr - cfg_region->region_base, *val); + break; + case DEBUGFS_READ64: + val_l = RREG32(addr - cfg_region->region_base); + val_h = RREG32(addr + sizeof(u32) - cfg_region->region_base); + + *val = (((u64) val_h) << 32) | val_l; + break; + case DEBUGFS_WRITE64: + WREG32(addr - cfg_region->region_base, lower_32_bits(*val)); + WREG32(addr + sizeof(u32) - cfg_region->region_base, upper_32_bits(*val)); + break; + default: + dev_err(hdev->dev, "access type %d is not supported\n", acc_type); + return -EOPNOTSUPP; + } + + return 0; +} + +/* + * hl_access_dev_mem - access device memory + * + * @hdev: pointer to habanalabs device structure + * @region_type: the type of the region the address belongs to + * @addr: the address to access + * @val: the value to write from or read to + * @acc_type: the type of access (r/w, 32/64) + */ +int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, + u64 addr, u64 *val, enum debugfs_access_type acc_type) +{ + switch (region_type) { + case PCI_REGION_CFG: + return hl_access_cfg_region(hdev, addr, val, acc_type); + case PCI_REGION_SRAM: + case PCI_REGION_DRAM: + return hl_access_sram_dram_region(hdev, addr, val, acc_type, + region_type); + default: + return -EFAULT; + } + + return 0; +} + +void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...) +{ + va_list args; + int str_size; + + va_start(args, fmt); + /* Calculate formatted string length. Assuming each string is null terminated, hence + * increment result by 1 + */ + str_size = vsnprintf(NULL, 0, fmt, args) + 1; + va_end(args); + + if ((e->actual_size + str_size) < e->allocated_buf_size) { + va_start(args, fmt); + vsnprintf(e->buf + e->actual_size, str_size, fmt, args); + va_end(args); + } + + /* Need to update the size even when not updating destination buffer to get the exact size + * of all input strings + */ + e->actual_size += str_size; +} + +enum hl_device_status hl_device_status(struct hl_device *hdev) +{ + enum hl_device_status status; + + if (hdev->reset_info.in_reset) { + if (hdev->reset_info.in_compute_reset) + status = HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE; + else + status = HL_DEVICE_STATUS_IN_RESET; + } else if (hdev->reset_info.needs_reset) { + status = HL_DEVICE_STATUS_NEEDS_RESET; + } else if (hdev->disabled) { + status = HL_DEVICE_STATUS_MALFUNCTION; + } else if (!hdev->init_done) { + status = HL_DEVICE_STATUS_IN_DEVICE_CREATION; + } else { + status = HL_DEVICE_STATUS_OPERATIONAL; + } + + return status; +} + +bool hl_device_operational(struct hl_device *hdev, + enum hl_device_status *status) +{ + enum hl_device_status current_status; + + current_status = hl_device_status(hdev); + if (status) + *status = current_status; + + switch (current_status) { + case HL_DEVICE_STATUS_IN_RESET: + case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE: + case HL_DEVICE_STATUS_MALFUNCTION: + case HL_DEVICE_STATUS_NEEDS_RESET: + return false; + case HL_DEVICE_STATUS_OPERATIONAL: + case HL_DEVICE_STATUS_IN_DEVICE_CREATION: + default: + return true; + } +} + +static void hpriv_release(struct kref *ref) +{ + u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; + bool device_is_idle = true; + struct hl_fpriv *hpriv; + struct hl_device *hdev; + + hpriv = container_of(ref, struct hl_fpriv, refcount); + + hdev = hpriv->hdev; + + hdev->asic_funcs->send_device_activity(hdev, false); + + put_pid(hpriv->taskpid); + + hl_debugfs_remove_file(hpriv); + + mutex_destroy(&hpriv->ctx_lock); + mutex_destroy(&hpriv->restore_phase_mutex); + + if ((!hdev->pldm) && (hdev->pdev) && + (!hdev->asic_funcs->is_device_idle(hdev, + idle_mask, + HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) { + dev_err(hdev->dev, + "device not idle after user context is closed (0x%llx_%llx)\n", + idle_mask[1], idle_mask[0]); + + device_is_idle = false; + } + + /* We need to remove the user from the list to make sure the reset process won't + * try to kill the user process. Because, if we got here, it means there are no + * more driver/device resources that the user process is occupying so there is + * no need to kill it + * + * However, we can't set the compute_ctx to NULL at this stage. This is to prevent + * a race between the release and opening the device again. We don't want to let + * a user open the device while there a reset is about to happen. + */ + mutex_lock(&hdev->fpriv_list_lock); + list_del(&hpriv->dev_node); + mutex_unlock(&hdev->fpriv_list_lock); + + if (!device_is_idle || hdev->reset_upon_device_release) { + hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE); + } else { + int rc = hdev->asic_funcs->scrub_device_mem(hdev); + + if (rc) + dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc); + } + + /* Now we can mark the compute_ctx as not active. Even if a reset is running in a different + * thread, we don't care because the in_reset is marked so if a user will try to open + * the device it will fail on that, even if compute_ctx is false. + */ + mutex_lock(&hdev->fpriv_list_lock); + hdev->is_compute_ctx_active = false; + mutex_unlock(&hdev->fpriv_list_lock); + + hdev->compute_ctx_in_release = 0; + + /* release the eventfd */ + if (hpriv->notifier_event.eventfd) + eventfd_ctx_put(hpriv->notifier_event.eventfd); + + mutex_destroy(&hpriv->notifier_event.lock); + + kfree(hpriv); +} + +void hl_hpriv_get(struct hl_fpriv *hpriv) +{ + kref_get(&hpriv->refcount); +} + +int hl_hpriv_put(struct hl_fpriv *hpriv) +{ + return kref_put(&hpriv->refcount, hpriv_release); +} + +/* + * hl_device_release - release function for habanalabs device + * + * @inode: pointer to inode structure + * @filp: pointer to file structure + * + * Called when process closes an habanalabs device + */ +static int hl_device_release(struct inode *inode, struct file *filp) +{ + struct hl_fpriv *hpriv = filp->private_data; + struct hl_device *hdev = hpriv->hdev; + + filp->private_data = NULL; + + if (!hdev) { + pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n"); + put_pid(hpriv->taskpid); + return 0; + } + + /* Each pending user interrupt holds the user's context, hence we + * must release them all before calling hl_ctx_mgr_fini(). + */ + hl_release_pending_user_interrupts(hpriv->hdev); + + hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr); + hl_mem_mgr_fini(&hpriv->mem_mgr); + + hdev->compute_ctx_in_release = 1; + + if (!hl_hpriv_put(hpriv)) + dev_notice(hdev->dev, + "User process closed FD but device still in use\n"); + + hdev->last_open_session_duration_jif = + jiffies - hdev->last_successful_open_jif; + + return 0; +} + +static int hl_device_release_ctrl(struct inode *inode, struct file *filp) +{ + struct hl_fpriv *hpriv = filp->private_data; + struct hl_device *hdev = hpriv->hdev; + + filp->private_data = NULL; + + if (!hdev) { + pr_err("Closing FD after device was removed\n"); + goto out; + } + + mutex_lock(&hdev->fpriv_ctrl_list_lock); + list_del(&hpriv->dev_node); + mutex_unlock(&hdev->fpriv_ctrl_list_lock); +out: + /* release the eventfd */ + if (hpriv->notifier_event.eventfd) + eventfd_ctx_put(hpriv->notifier_event.eventfd); + + mutex_destroy(&hpriv->notifier_event.lock); + put_pid(hpriv->taskpid); + + kfree(hpriv); + + return 0; +} + +/* + * hl_mmap - mmap function for habanalabs device + * + * @*filp: pointer to file structure + * @*vma: pointer to vm_area_struct of the process + * + * Called when process does an mmap on habanalabs device. Call the relevant mmap + * function at the end of the common code. + */ +static int hl_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct hl_fpriv *hpriv = filp->private_data; + struct hl_device *hdev = hpriv->hdev; + unsigned long vm_pgoff; + + if (!hdev) { + pr_err_ratelimited("Trying to mmap after device was removed! Please close FD\n"); + return -ENODEV; + } + + vm_pgoff = vma->vm_pgoff; + + switch (vm_pgoff & HL_MMAP_TYPE_MASK) { + case HL_MMAP_TYPE_BLOCK: + vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff); + return hl_hw_block_mmap(hpriv, vma); + + case HL_MMAP_TYPE_CB: + case HL_MMAP_TYPE_TS_BUFF: + return hl_mem_mgr_mmap(&hpriv->mem_mgr, vma, NULL); + } + return -EINVAL; +} + +static const struct file_operations hl_ops = { + .owner = THIS_MODULE, + .open = hl_device_open, + .release = hl_device_release, + .mmap = hl_mmap, + .unlocked_ioctl = hl_ioctl, + .compat_ioctl = hl_ioctl +}; + +static const struct file_operations hl_ctrl_ops = { + .owner = THIS_MODULE, + .open = hl_device_open_ctrl, + .release = hl_device_release_ctrl, + .unlocked_ioctl = hl_ioctl_control, + .compat_ioctl = hl_ioctl_control +}; + +static void device_release_func(struct device *dev) +{ + kfree(dev); +} + +/* + * device_init_cdev - Initialize cdev and device for habanalabs device + * + * @hdev: pointer to habanalabs device structure + * @hclass: pointer to the class object of the device + * @minor: minor number of the specific device + * @fpos: file operations to install for this device + * @name: name of the device as it will appear in the filesystem + * @cdev: pointer to the char device object that will be initialized + * @dev: pointer to the device object that will be initialized + * + * Initialize a cdev and a Linux device for habanalabs's device. + */ +static int device_init_cdev(struct hl_device *hdev, struct class *hclass, + int minor, const struct file_operations *fops, + char *name, struct cdev *cdev, + struct device **dev) +{ + cdev_init(cdev, fops); + cdev->owner = THIS_MODULE; + + *dev = kzalloc(sizeof(**dev), GFP_KERNEL); + if (!*dev) + return -ENOMEM; + + device_initialize(*dev); + (*dev)->devt = MKDEV(hdev->major, minor); + (*dev)->class = hclass; + (*dev)->release = device_release_func; + dev_set_drvdata(*dev, hdev); + dev_set_name(*dev, "%s", name); + + return 0; +} + +static int device_cdev_sysfs_add(struct hl_device *hdev) +{ + int rc; + + rc = cdev_device_add(&hdev->cdev, hdev->dev); + if (rc) { + dev_err(hdev->dev, + "failed to add a char device to the system\n"); + return rc; + } + + rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl); + if (rc) { + dev_err(hdev->dev, + "failed to add a control char device to the system\n"); + goto delete_cdev_device; + } + + /* hl_sysfs_init() must be done after adding the device to the system */ + rc = hl_sysfs_init(hdev); + if (rc) { + dev_err(hdev->dev, "failed to initialize sysfs\n"); + goto delete_ctrl_cdev_device; + } + + hdev->cdev_sysfs_created = true; + + return 0; + +delete_ctrl_cdev_device: + cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); +delete_cdev_device: + cdev_device_del(&hdev->cdev, hdev->dev); + return rc; +} + +static void device_cdev_sysfs_del(struct hl_device *hdev) +{ + if (!hdev->cdev_sysfs_created) + goto put_devices; + + hl_sysfs_fini(hdev); + cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl); + cdev_device_del(&hdev->cdev, hdev->dev); + +put_devices: + put_device(hdev->dev); + put_device(hdev->dev_ctrl); +} + +static void device_hard_reset_pending(struct work_struct *work) +{ + struct hl_device_reset_work *device_reset_work = + container_of(work, struct hl_device_reset_work, reset_work.work); + struct hl_device *hdev = device_reset_work->hdev; + u32 flags; + int rc; + + flags = device_reset_work->flags | HL_DRV_RESET_FROM_RESET_THR; + + rc = hl_device_reset(hdev, flags); + if ((rc == -EBUSY) && !hdev->device_fini_pending) { + dev_info(hdev->dev, + "Could not reset device. will try again in %u seconds", + HL_PENDING_RESET_PER_SEC); + + queue_delayed_work(device_reset_work->wq, + &device_reset_work->reset_work, + msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000)); + } +} + +/* + * device_early_init - do some early initialization for the habanalabs device + * + * @hdev: pointer to habanalabs device structure + * + * Install the relevant function pointers and call the early_init function, + * if such a function exists + */ +static int device_early_init(struct hl_device *hdev) +{ + int i, rc; + char workq_name[32]; + + switch (hdev->asic_type) { + case ASIC_GOYA: + goya_set_asic_funcs(hdev); + strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name)); + break; + case ASIC_GAUDI: + gaudi_set_asic_funcs(hdev); + strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name)); + break; + case ASIC_GAUDI_SEC: + gaudi_set_asic_funcs(hdev); + strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name)); + break; + case ASIC_GAUDI2: + gaudi2_set_asic_funcs(hdev); + strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name)); + break; + case ASIC_GAUDI2_SEC: + gaudi2_set_asic_funcs(hdev); + strscpy(hdev->asic_name, "GAUDI2 SEC", sizeof(hdev->asic_name)); + break; + default: + dev_err(hdev->dev, "Unrecognized ASIC type %d\n", + hdev->asic_type); + return -EINVAL; + } + + rc = hdev->asic_funcs->early_init(hdev); + if (rc) + return rc; + + rc = hl_asid_init(hdev); + if (rc) + goto early_fini; + + if (hdev->asic_prop.completion_queues_count) { + hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count, + sizeof(struct workqueue_struct *), + GFP_KERNEL); + if (!hdev->cq_wq) { + rc = -ENOMEM; + goto asid_fini; + } + } + + for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { + snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i); + hdev->cq_wq[i] = create_singlethread_workqueue(workq_name); + if (hdev->cq_wq[i] == NULL) { + dev_err(hdev->dev, "Failed to allocate CQ workqueue\n"); + rc = -ENOMEM; + goto free_cq_wq; + } + } + + hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0); + if (hdev->eq_wq == NULL) { + dev_err(hdev->dev, "Failed to allocate EQ workqueue\n"); + rc = -ENOMEM; + goto free_cq_wq; + } + + hdev->cs_cmplt_wq = alloc_workqueue("hl-cs-completions", WQ_UNBOUND, 0); + if (!hdev->cs_cmplt_wq) { + dev_err(hdev->dev, + "Failed to allocate CS completions workqueue\n"); + rc = -ENOMEM; + goto free_eq_wq; + } + + hdev->ts_free_obj_wq = alloc_workqueue("hl-ts-free-obj", WQ_UNBOUND, 0); + if (!hdev->ts_free_obj_wq) { + dev_err(hdev->dev, + "Failed to allocate Timestamp registration free workqueue\n"); + rc = -ENOMEM; + goto free_cs_cmplt_wq; + } + + hdev->pf_wq = alloc_workqueue("hl-prefetch", WQ_UNBOUND, 0); + if (!hdev->pf_wq) { + dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n"); + rc = -ENOMEM; + goto free_ts_free_wq; + } + + hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), + GFP_KERNEL); + if (!hdev->hl_chip_info) { + rc = -ENOMEM; + goto free_pf_wq; + } + + rc = hl_mmu_if_set_funcs(hdev); + if (rc) + goto free_chip_info; + + hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr); + + hdev->device_reset_work.wq = + create_singlethread_workqueue("hl_device_reset"); + if (!hdev->device_reset_work.wq) { + rc = -ENOMEM; + dev_err(hdev->dev, "Failed to create device reset WQ\n"); + goto free_cb_mgr; + } + + INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, + device_hard_reset_pending); + hdev->device_reset_work.hdev = hdev; + hdev->device_fini_pending = 0; + + mutex_init(&hdev->send_cpu_message_lock); + mutex_init(&hdev->debug_lock); + INIT_LIST_HEAD(&hdev->cs_mirror_list); + spin_lock_init(&hdev->cs_mirror_lock); + spin_lock_init(&hdev->reset_info.lock); + INIT_LIST_HEAD(&hdev->fpriv_list); + INIT_LIST_HEAD(&hdev->fpriv_ctrl_list); + mutex_init(&hdev->fpriv_list_lock); + mutex_init(&hdev->fpriv_ctrl_list_lock); + mutex_init(&hdev->clk_throttling.lock); + + return 0; + +free_cb_mgr: + hl_mem_mgr_fini(&hdev->kernel_mem_mgr); +free_chip_info: + kfree(hdev->hl_chip_info); +free_pf_wq: + destroy_workqueue(hdev->pf_wq); +free_ts_free_wq: + destroy_workqueue(hdev->ts_free_obj_wq); +free_cs_cmplt_wq: + destroy_workqueue(hdev->cs_cmplt_wq); +free_eq_wq: + destroy_workqueue(hdev->eq_wq); +free_cq_wq: + for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) + if (hdev->cq_wq[i]) + destroy_workqueue(hdev->cq_wq[i]); + kfree(hdev->cq_wq); +asid_fini: + hl_asid_fini(hdev); +early_fini: + if (hdev->asic_funcs->early_fini) + hdev->asic_funcs->early_fini(hdev); + + return rc; +} + +/* + * device_early_fini - finalize all that was done in device_early_init + * + * @hdev: pointer to habanalabs device structure + * + */ +static void device_early_fini(struct hl_device *hdev) +{ + int i; + + mutex_destroy(&hdev->debug_lock); + mutex_destroy(&hdev->send_cpu_message_lock); + + mutex_destroy(&hdev->fpriv_list_lock); + mutex_destroy(&hdev->fpriv_ctrl_list_lock); + + mutex_destroy(&hdev->clk_throttling.lock); + + hl_mem_mgr_fini(&hdev->kernel_mem_mgr); + + kfree(hdev->hl_chip_info); + + destroy_workqueue(hdev->pf_wq); + destroy_workqueue(hdev->ts_free_obj_wq); + destroy_workqueue(hdev->cs_cmplt_wq); + destroy_workqueue(hdev->eq_wq); + destroy_workqueue(hdev->device_reset_work.wq); + + for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) + destroy_workqueue(hdev->cq_wq[i]); + kfree(hdev->cq_wq); + + hl_asid_fini(hdev); + + if (hdev->asic_funcs->early_fini) + hdev->asic_funcs->early_fini(hdev); +} + +static bool is_pci_link_healthy(struct hl_device *hdev) +{ + u16 vendor_id; + + if (!hdev->pdev) + return false; + + pci_read_config_word(hdev->pdev, PCI_VENDOR_ID, &vendor_id); + + return (vendor_id == PCI_VENDOR_ID_HABANALABS); +} + +static void hl_device_heartbeat(struct work_struct *work) +{ + struct hl_device *hdev = container_of(work, struct hl_device, + work_heartbeat.work); + + if (!hl_device_operational(hdev, NULL)) + goto reschedule; + + if (!hdev->asic_funcs->send_heartbeat(hdev)) + goto reschedule; + + if (hl_device_operational(hdev, NULL)) + dev_err(hdev->dev, "Device heartbeat failed! PCI link is %s\n", + is_pci_link_healthy(hdev) ? "healthy" : "broken"); + + hl_device_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT); + + return; + +reschedule: + /* + * prev_reset_trigger tracks consecutive fatal h/w errors until first + * heartbeat immediately post reset. + * If control reached here, then at least one heartbeat work has been + * scheduled since last reset/init cycle. + * So if the device is not already in reset cycle, reset the flag + * prev_reset_trigger as no reset occurred with HL_DRV_RESET_FW_FATAL_ERR + * status for at least one heartbeat. From this point driver restarts + * tracking future consecutive fatal errors. + */ + if (!hdev->reset_info.in_reset) + hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT; + + schedule_delayed_work(&hdev->work_heartbeat, + usecs_to_jiffies(HL_HEARTBEAT_PER_USEC)); +} + +/* + * device_late_init - do late stuff initialization for the habanalabs device + * + * @hdev: pointer to habanalabs device structure + * + * Do stuff that either needs the device H/W queues to be active or needs + * to happen after all the rest of the initialization is finished + */ +static int device_late_init(struct hl_device *hdev) +{ + int rc; + + if (hdev->asic_funcs->late_init) { + rc = hdev->asic_funcs->late_init(hdev); + if (rc) { + dev_err(hdev->dev, + "failed late initialization for the H/W\n"); + return rc; + } + } + + hdev->high_pll = hdev->asic_prop.high_pll; + + if (hdev->heartbeat) { + INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat); + schedule_delayed_work(&hdev->work_heartbeat, + usecs_to_jiffies(HL_HEARTBEAT_PER_USEC)); + } + + hdev->late_init_done = true; + + return 0; +} + +/* + * device_late_fini - finalize all that was done in device_late_init + * + * @hdev: pointer to habanalabs device structure + * + */ +static void device_late_fini(struct hl_device *hdev) +{ + if (!hdev->late_init_done) + return; + + if (hdev->heartbeat) + cancel_delayed_work_sync(&hdev->work_heartbeat); + + if (hdev->asic_funcs->late_fini) + hdev->asic_funcs->late_fini(hdev); + + hdev->late_init_done = false; +} + +int hl_device_utilization(struct hl_device *hdev, u32 *utilization) +{ + u64 max_power, curr_power, dc_power, dividend; + int rc; + + max_power = hdev->max_power; + dc_power = hdev->asic_prop.dc_power_default; + rc = hl_fw_cpucp_power_get(hdev, &curr_power); + + if (rc) + return rc; + + curr_power = clamp(curr_power, dc_power, max_power); + + dividend = (curr_power - dc_power) * 100; + *utilization = (u32) div_u64(dividend, (max_power - dc_power)); + + return 0; +} + +int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable) +{ + int rc = 0; + + mutex_lock(&hdev->debug_lock); + + if (!enable) { + if (!hdev->in_debug) { + dev_err(hdev->dev, + "Failed to disable debug mode because device was not in debug mode\n"); + rc = -EFAULT; + goto out; + } + + if (!hdev->reset_info.hard_reset_pending) + hdev->asic_funcs->halt_coresight(hdev, ctx); + + hdev->in_debug = 0; + + goto out; + } + + if (hdev->in_debug) { + dev_err(hdev->dev, + "Failed to enable debug mode because device is already in debug mode\n"); + rc = -EFAULT; + goto out; + } + + hdev->in_debug = 1; + +out: + mutex_unlock(&hdev->debug_lock); + + return rc; +} + +static void take_release_locks(struct hl_device *hdev) +{ + /* Flush anyone that is inside the critical section of enqueue + * jobs to the H/W + */ + hdev->asic_funcs->hw_queues_lock(hdev); + hdev->asic_funcs->hw_queues_unlock(hdev); + + /* Flush processes that are sending message to CPU */ + mutex_lock(&hdev->send_cpu_message_lock); + mutex_unlock(&hdev->send_cpu_message_lock); + + /* Flush anyone that is inside device open */ + mutex_lock(&hdev->fpriv_list_lock); + mutex_unlock(&hdev->fpriv_list_lock); + mutex_lock(&hdev->fpriv_ctrl_list_lock); + mutex_unlock(&hdev->fpriv_ctrl_list_lock); +} + +static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset, + bool skip_wq_flush) +{ + if (hard_reset) + device_late_fini(hdev); + + /* + * Halt the engines and disable interrupts so we won't get any more + * completions from H/W and we won't have any accesses from the + * H/W to the host machine + */ + hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset); + + /* Go over all the queues, release all CS and their jobs */ + hl_cs_rollback_all(hdev, skip_wq_flush); + + /* flush the MMU prefetch workqueue */ + flush_workqueue(hdev->pf_wq); + + /* Release all pending user interrupts, each pending user interrupt + * holds a reference to user context + */ + hl_release_pending_user_interrupts(hdev); +} + +/* + * hl_device_suspend - initiate device suspend + * + * @hdev: pointer to habanalabs device structure + * + * Puts the hw in the suspend state (all asics). + * Returns 0 for success or an error on failure. + * Called at driver suspend. + */ +int hl_device_suspend(struct hl_device *hdev) +{ + int rc; + + pci_save_state(hdev->pdev); + + /* Block future CS/VM/JOB completion operations */ + spin_lock(&hdev->reset_info.lock); + if (hdev->reset_info.in_reset) { + spin_unlock(&hdev->reset_info.lock); + dev_err(hdev->dev, "Can't suspend while in reset\n"); + return -EIO; + } + hdev->reset_info.in_reset = 1; + spin_unlock(&hdev->reset_info.lock); + + /* This blocks all other stuff that is not blocked by in_reset */ + hdev->disabled = true; + + take_release_locks(hdev); + + rc = hdev->asic_funcs->suspend(hdev); + if (rc) + dev_err(hdev->dev, + "Failed to disable PCI access of device CPU\n"); + + /* Shut down the device */ + pci_disable_device(hdev->pdev); + pci_set_power_state(hdev->pdev, PCI_D3hot); + + return 0; +} + +/* + * hl_device_resume - initiate device resume + * + * @hdev: pointer to habanalabs device structure + * + * Bring the hw back to operating state (all asics). + * Returns 0 for success or an error on failure. + * Called at driver resume. + */ +int hl_device_resume(struct hl_device *hdev) +{ + int rc; + + pci_set_power_state(hdev->pdev, PCI_D0); + pci_restore_state(hdev->pdev); + rc = pci_enable_device_mem(hdev->pdev); + if (rc) { + dev_err(hdev->dev, + "Failed to enable PCI device in resume\n"); + return rc; + } + + pci_set_master(hdev->pdev); + + rc = hdev->asic_funcs->resume(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to resume device after suspend\n"); + goto disable_device; + } + + + /* 'in_reset' was set to true during suspend, now we must clear it in order + * for hard reset to be performed + */ + spin_lock(&hdev->reset_info.lock); + hdev->reset_info.in_reset = 0; + spin_unlock(&hdev->reset_info.lock); + + rc = hl_device_reset(hdev, HL_DRV_RESET_HARD); + if (rc) { + dev_err(hdev->dev, "Failed to reset device during resume\n"); + goto disable_device; + } + + return 0; + +disable_device: + pci_clear_master(hdev->pdev); + pci_disable_device(hdev->pdev); + + return rc; +} + +static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev) +{ + struct task_struct *task = NULL; + struct list_head *fd_list; + struct hl_fpriv *hpriv; + struct mutex *fd_lock; + u32 pending_cnt; + + fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock; + fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list; + + /* Giving time for user to close FD, and for processes that are inside + * hl_device_open to finish + */ + if (!list_empty(fd_list)) + ssleep(1); + + if (timeout) { + pending_cnt = timeout; + } else { + if (hdev->process_kill_trial_cnt) { + /* Processes have been already killed */ + pending_cnt = 1; + goto wait_for_processes; + } else { + /* Wait a small period after process kill */ + pending_cnt = HL_PENDING_RESET_PER_SEC; + } + } + + mutex_lock(fd_lock); + + /* This section must be protected because we are dereferencing + * pointers that are freed if the process exits + */ + list_for_each_entry(hpriv, fd_list, dev_node) { + task = get_pid_task(hpriv->taskpid, PIDTYPE_PID); + if (task) { + dev_info(hdev->dev, "Killing user process pid=%d\n", + task_pid_nr(task)); + send_sig(SIGKILL, task, 1); + usleep_range(1000, 10000); + + put_task_struct(task); + } else { + /* + * If we got here, it means that process was killed from outside the driver + * right after it started looping on fd_list and before get_pid_task, thus + * we don't need to kill it. + */ + dev_dbg(hdev->dev, + "Can't get task struct for user process, assuming process was killed from outside the driver\n"); + } + } + + mutex_unlock(fd_lock); + + /* + * We killed the open users, but that doesn't mean they are closed. + * It could be that they are running a long cleanup phase in the driver + * e.g. MMU unmappings, or running other long teardown flow even before + * our cleanup. + * Therefore we need to wait again to make sure they are closed before + * continuing with the reset. + */ + +wait_for_processes: + while ((!list_empty(fd_list)) && (pending_cnt)) { + dev_dbg(hdev->dev, + "Waiting for all unmap operations to finish before hard reset\n"); + + pending_cnt--; + + ssleep(1); + } + + /* All processes exited successfully */ + if (list_empty(fd_list)) + return 0; + + /* Give up waiting for processes to exit */ + if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS) + return -ETIME; + + hdev->process_kill_trial_cnt++; + + return -EBUSY; +} + +static void device_disable_open_processes(struct hl_device *hdev, bool control_dev) +{ + struct list_head *fd_list; + struct hl_fpriv *hpriv; + struct mutex *fd_lock; + + fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock; + fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list; + + mutex_lock(fd_lock); + list_for_each_entry(hpriv, fd_list, dev_node) + hpriv->hdev = NULL; + mutex_unlock(fd_lock); +} + +static void handle_reset_trigger(struct hl_device *hdev, u32 flags) +{ + u32 cur_reset_trigger = HL_RESET_TRIGGER_DEFAULT; + + /* + * 'reset cause' is being updated here, because getting here + * means that it's the 1st time and the last time we're here + * ('in_reset' makes sure of it). This makes sure that + * 'reset_cause' will continue holding its 1st recorded reason! + */ + if (flags & HL_DRV_RESET_HEARTBEAT) { + hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT; + cur_reset_trigger = HL_DRV_RESET_HEARTBEAT; + } else if (flags & HL_DRV_RESET_TDR) { + hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR; + cur_reset_trigger = HL_DRV_RESET_TDR; + } else if (flags & HL_DRV_RESET_FW_FATAL_ERR) { + hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; + cur_reset_trigger = HL_DRV_RESET_FW_FATAL_ERR; + } else { + hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN; + } + + /* + * If reset cause is same twice, then reset_trigger_repeated + * is set and if this reset is due to a fatal FW error + * device is set to an unstable state. + */ + if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) { + hdev->reset_info.prev_reset_trigger = cur_reset_trigger; + hdev->reset_info.reset_trigger_repeated = 0; + } else { + hdev->reset_info.reset_trigger_repeated = 1; + } + + /* If reset is due to heartbeat, device CPU is no responsive in + * which case no point sending PCI disable message to it. + * + * If F/W is performing the reset, no need to send it a message to disable + * PCI access + */ + if ((flags & HL_DRV_RESET_HARD) && + !(flags & (HL_DRV_RESET_HEARTBEAT | HL_DRV_RESET_BYPASS_REQ_TO_FW))) { + /* Disable PCI access from device F/W so he won't send + * us additional interrupts. We disable MSI/MSI-X at + * the halt_engines function and we can't have the F/W + * sending us interrupts after that. We need to disable + * the access here because if the device is marked + * disable, the message won't be send. Also, in case + * of heartbeat, the device CPU is marked as disable + * so this message won't be sent + */ + if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) + dev_warn(hdev->dev, + "Failed to disable PCI access by F/W\n"); + } +} + +/* + * hl_device_reset - reset the device + * + * @hdev: pointer to habanalabs device structure + * @flags: reset flags. + * + * Block future CS and wait for pending CS to be enqueued + * Call ASIC H/W fini + * Flush all completions + * Re-initialize all internal data structures + * Call ASIC H/W init, late_init + * Test queues + * Enable device + * + * Returns 0 for success or an error on failure. + */ +int hl_device_reset(struct hl_device *hdev, u32 flags) +{ + bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false, + reset_upon_device_release = false, schedule_hard_reset = false, + skip_wq_flush, delay_reset; + u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0}; + struct hl_ctx *ctx; + int i, rc; + + if (!hdev->init_done) { + dev_err(hdev->dev, "Can't reset before initialization is done\n"); + return 0; + } + + hard_reset = !!(flags & HL_DRV_RESET_HARD); + from_hard_reset_thread = !!(flags & HL_DRV_RESET_FROM_RESET_THR); + fw_reset = !!(flags & HL_DRV_RESET_BYPASS_REQ_TO_FW); + skip_wq_flush = !!(flags & HL_DRV_RESET_DEV_RELEASE); + delay_reset = !!(flags & HL_DRV_RESET_DELAY); + + if (!hard_reset && !hdev->asic_prop.supports_compute_reset) { + hard_instead_soft = true; + hard_reset = true; + } + + if (hdev->reset_upon_device_release && (flags & HL_DRV_RESET_DEV_RELEASE)) { + if (hard_reset) { + dev_crit(hdev->dev, + "Aborting reset because hard-reset is mutually exclusive with reset-on-device-release\n"); + return -EINVAL; + } + + reset_upon_device_release = true; + + goto do_reset; + } + + if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) { + hard_instead_soft = true; + hard_reset = true; + } + + if (hard_instead_soft) + dev_dbg(hdev->dev, "Doing hard-reset instead of compute reset\n"); + +do_reset: + /* Re-entry of reset thread */ + if (from_hard_reset_thread && hdev->process_kill_trial_cnt) + goto kill_processes; + + /* + * Prevent concurrency in this function - only one reset should be + * done at any given time. Only need to perform this if we didn't + * get from the dedicated hard reset thread + */ + if (!from_hard_reset_thread) { + /* Block future CS/VM/JOB completion operations */ + spin_lock(&hdev->reset_info.lock); + if (hdev->reset_info.in_reset) { + /* We only allow scheduling of a hard reset during compute reset */ + if (hard_reset && hdev->reset_info.in_compute_reset) + hdev->reset_info.hard_reset_schedule_flags = flags; + spin_unlock(&hdev->reset_info.lock); + return 0; + } + + /* This still allows the completion of some KDMA ops + * Update this before in_reset because in_compute_reset implies we are in reset + */ + hdev->reset_info.in_compute_reset = !hard_reset; + + hdev->reset_info.in_reset = 1; + + spin_unlock(&hdev->reset_info.lock); + + if (delay_reset) + usleep_range(HL_RESET_DELAY_USEC, HL_RESET_DELAY_USEC << 1); + + handle_reset_trigger(hdev, flags); + + /* This also blocks future CS/VM/JOB completion operations */ + hdev->disabled = true; + + take_release_locks(hdev); + + if (hard_reset) + dev_info(hdev->dev, "Going to reset device\n"); + else if (reset_upon_device_release) + dev_dbg(hdev->dev, "Going to reset device after release by user\n"); + else + dev_dbg(hdev->dev, "Going to reset engines of inference device\n"); + } + +again: + if ((hard_reset) && (!from_hard_reset_thread)) { + hdev->reset_info.hard_reset_pending = true; + + hdev->process_kill_trial_cnt = 0; + + hdev->device_reset_work.flags = flags; + + /* + * Because the reset function can't run from heartbeat work, + * we need to call the reset function from a dedicated work. + */ + queue_delayed_work(hdev->device_reset_work.wq, + &hdev->device_reset_work.reset_work, 0); + + return 0; + } + + cleanup_resources(hdev, hard_reset, fw_reset, skip_wq_flush); + +kill_processes: + if (hard_reset) { + /* Kill processes here after CS rollback. This is because the + * process can't really exit until all its CSs are done, which + * is what we do in cs rollback + */ + rc = device_kill_open_processes(hdev, 0, false); + + if (rc == -EBUSY) { + if (hdev->device_fini_pending) { + dev_crit(hdev->dev, + "%s Failed to kill all open processes, stopping hard reset\n", + dev_name(&(hdev)->pdev->dev)); + goto out_err; + } + + /* signal reset thread to reschedule */ + return rc; + } + + if (rc) { + dev_crit(hdev->dev, + "%s Failed to kill all open processes, stopping hard reset\n", + dev_name(&(hdev)->pdev->dev)); + goto out_err; + } + + /* Flush the Event queue workers to make sure no other thread is + * reading or writing to registers during the reset + */ + flush_workqueue(hdev->eq_wq); + } + + /* Reset the H/W. It will be in idle state after this returns */ + hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset); + + if (hard_reset) { + hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE; + + /* Release kernel context */ + if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1) + hdev->kernel_ctx = NULL; + + hl_vm_fini(hdev); + hl_mmu_fini(hdev); + hl_eq_reset(hdev, &hdev->event_queue); + } + + /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */ + hl_hw_queue_reset(hdev, hard_reset); + for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) + hl_cq_reset(hdev, &hdev->completion_queue[i]); + + /* Make sure the context switch phase will run again */ + ctx = hl_get_compute_ctx(hdev); + if (ctx) { + atomic_set(&ctx->thread_ctx_switch_token, 1); + ctx->thread_ctx_switch_wait_token = 0; + hl_ctx_put(ctx); + } + + /* Finished tear-down, starting to re-initialize */ + + if (hard_reset) { + hdev->device_cpu_disabled = false; + hdev->reset_info.hard_reset_pending = false; + + if (hdev->reset_info.reset_trigger_repeated && + (hdev->reset_info.prev_reset_trigger == + HL_DRV_RESET_FW_FATAL_ERR)) { + /* if there 2 back to back resets from FW, + * ensure driver puts the driver in a unusable state + */ + dev_crit(hdev->dev, + "%s Consecutive FW fatal errors received, stopping hard reset\n", + dev_name(&(hdev)->pdev->dev)); + rc = -EIO; + goto out_err; + } + + if (hdev->kernel_ctx) { + dev_crit(hdev->dev, + "%s kernel ctx was alive during hard reset, something is terribly wrong\n", + dev_name(&(hdev)->pdev->dev)); + rc = -EBUSY; + goto out_err; + } + + rc = hl_mmu_init(hdev); + if (rc) { + dev_err(hdev->dev, + "Failed to initialize MMU S/W after hard reset\n"); + goto out_err; + } + + /* Allocate the kernel context */ + hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), + GFP_KERNEL); + if (!hdev->kernel_ctx) { + rc = -ENOMEM; + hl_mmu_fini(hdev); + goto out_err; + } + + hdev->is_compute_ctx_active = false; + + rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); + if (rc) { + dev_err(hdev->dev, + "failed to init kernel ctx in hard reset\n"); + kfree(hdev->kernel_ctx); + hdev->kernel_ctx = NULL; + hl_mmu_fini(hdev); + goto out_err; + } + } + + /* Device is now enabled as part of the initialization requires + * communication with the device firmware to get information that + * is required for the initialization itself + */ + hdev->disabled = false; + + /* F/W security enabled indication might be updated after hard-reset */ + if (hard_reset) { + rc = hl_fw_read_preboot_status(hdev); + if (rc) + goto out_err; + } + + rc = hdev->asic_funcs->hw_init(hdev); + if (rc) { + dev_err(hdev->dev, "failed to initialize the H/W after reset\n"); + goto out_err; + } + + /* If device is not idle fail the reset process */ + if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask, + HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) { + dev_err(hdev->dev, "device is not idle (mask 0x%llx_%llx) after reset\n", + idle_mask[1], idle_mask[0]); + rc = -EIO; + goto out_err; + } + + /* Check that the communication with the device is working */ + rc = hdev->asic_funcs->test_queues(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to detect if device is alive after reset\n"); + goto out_err; + } + + if (hard_reset) { + rc = device_late_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed late init after hard reset\n"); + goto out_err; + } + + rc = hl_vm_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to init memory module after hard reset\n"); + goto out_err; + } + + if (!hdev->asic_prop.fw_security_enabled) + hl_fw_set_max_power(hdev); + } else { + rc = hdev->asic_funcs->compute_reset_late_init(hdev); + if (rc) { + if (reset_upon_device_release) + dev_err(hdev->dev, + "Failed late init in reset after device release\n"); + else + dev_err(hdev->dev, "Failed late init after compute reset\n"); + goto out_err; + } + } + + rc = hdev->asic_funcs->scrub_device_mem(hdev); + if (rc) { + dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc); + return rc; + } + + spin_lock(&hdev->reset_info.lock); + hdev->reset_info.in_compute_reset = 0; + + /* Schedule hard reset only if requested and if not already in hard reset. + * We keep 'in_reset' enabled, so no other reset can go in during the hard + * reset schedule + */ + if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags) + schedule_hard_reset = true; + else + hdev->reset_info.in_reset = 0; + + spin_unlock(&hdev->reset_info.lock); + + hdev->reset_info.needs_reset = false; + + if (hard_reset) + dev_info(hdev->dev, + "Successfully finished resetting the %s device\n", + dev_name(&(hdev)->pdev->dev)); + else + dev_dbg(hdev->dev, + "Successfully finished resetting the %s device\n", + dev_name(&(hdev)->pdev->dev)); + + if (hard_reset) { + hdev->reset_info.hard_reset_cnt++; + + /* After reset is done, we are ready to receive events from + * the F/W. We can't do it before because we will ignore events + * and if those events are fatal, we won't know about it and + * the device will be operational although it shouldn't be + */ + hdev->asic_funcs->enable_events_from_fw(hdev); + } else if (!reset_upon_device_release) { + hdev->reset_info.compute_reset_cnt++; + } + + if (schedule_hard_reset) { + dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n"); + flags = hdev->reset_info.hard_reset_schedule_flags; + hdev->reset_info.hard_reset_schedule_flags = 0; + hdev->disabled = true; + hard_reset = true; + handle_reset_trigger(hdev, flags); + goto again; + } + + return 0; + +out_err: + hdev->disabled = true; + + spin_lock(&hdev->reset_info.lock); + hdev->reset_info.in_compute_reset = 0; + + if (hard_reset) { + dev_err(hdev->dev, + "%s Failed to reset! Device is NOT usable\n", + dev_name(&(hdev)->pdev->dev)); + hdev->reset_info.hard_reset_cnt++; + } else if (reset_upon_device_release) { + spin_unlock(&hdev->reset_info.lock); + dev_err(hdev->dev, "Failed to reset device after user release\n"); + flags |= HL_DRV_RESET_HARD; + flags &= ~HL_DRV_RESET_DEV_RELEASE; + hard_reset = true; + goto again; + } else { + spin_unlock(&hdev->reset_info.lock); + dev_err(hdev->dev, "Failed to do compute reset\n"); + hdev->reset_info.compute_reset_cnt++; + flags |= HL_DRV_RESET_HARD; + hard_reset = true; + goto again; + } + + hdev->reset_info.in_reset = 0; + + spin_unlock(&hdev->reset_info.lock); + + return rc; +} + +static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event_mask) +{ + mutex_lock(¬ifier_event->lock); + notifier_event->events_mask |= event_mask; + + if (notifier_event->eventfd) + eventfd_signal(notifier_event->eventfd, 1); + + mutex_unlock(¬ifier_event->lock); +} + +/* + * hl_notifier_event_send_all - notify all user processes via eventfd + * + * @hdev: pointer to habanalabs device structure + * @event_mask: the occurred event/s + * Returns 0 for success or an error on failure. + */ +void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask) +{ + struct hl_fpriv *hpriv; + + mutex_lock(&hdev->fpriv_list_lock); + + list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) + hl_notifier_event_send(&hpriv->notifier_event, event_mask); + + mutex_unlock(&hdev->fpriv_list_lock); + + /* control device */ + mutex_lock(&hdev->fpriv_ctrl_list_lock); + + list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node) + hl_notifier_event_send(&hpriv->notifier_event, event_mask); + + mutex_unlock(&hdev->fpriv_ctrl_list_lock); +} + +/* + * hl_device_init - main initialization function for habanalabs device + * + * @hdev: pointer to habanalabs device structure + * + * Allocate an id for the device, do early initialization and then call the + * ASIC specific initialization functions. Finally, create the cdev and the + * Linux device to expose it to the user + */ +int hl_device_init(struct hl_device *hdev, struct class *hclass) +{ + int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt; + char *name; + bool add_cdev_sysfs_on_err = false; + + hdev->cdev_idx = hdev->id / 2; + + name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx); + if (!name) { + rc = -ENOMEM; + goto out_disabled; + } + + /* Initialize cdev and device structures */ + rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name, + &hdev->cdev, &hdev->dev); + + kfree(name); + + if (rc) + goto out_disabled; + + name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx); + if (!name) { + rc = -ENOMEM; + goto free_dev; + } + + /* Initialize cdev and device structures for control device */ + rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops, + name, &hdev->cdev_ctrl, &hdev->dev_ctrl); + + kfree(name); + + if (rc) + goto free_dev; + + /* Initialize ASIC function pointers and perform early init */ + rc = device_early_init(hdev); + if (rc) + goto free_dev_ctrl; + + user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count + + hdev->asic_prop.user_interrupt_count; + + if (user_interrupt_cnt) { + hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt), + GFP_KERNEL); + if (!hdev->user_interrupt) { + rc = -ENOMEM; + goto early_fini; + } + } + + /* + * Start calling ASIC initialization. First S/W then H/W and finally + * late init + */ + rc = hdev->asic_funcs->sw_init(hdev); + if (rc) + goto free_usr_intr_mem; + + + /* initialize completion structure for multi CS wait */ + hl_multi_cs_completion_init(hdev); + + /* + * Initialize the H/W queues. Must be done before hw_init, because + * there the addresses of the kernel queue are being written to the + * registers of the device + */ + rc = hl_hw_queues_create(hdev); + if (rc) { + dev_err(hdev->dev, "failed to initialize kernel queues\n"); + goto sw_fini; + } + + cq_cnt = hdev->asic_prop.completion_queues_count; + + /* + * Initialize the completion queues. Must be done before hw_init, + * because there the addresses of the completion queues are being + * passed as arguments to request_irq + */ + if (cq_cnt) { + hdev->completion_queue = kcalloc(cq_cnt, + sizeof(*hdev->completion_queue), + GFP_KERNEL); + + if (!hdev->completion_queue) { + dev_err(hdev->dev, + "failed to allocate completion queues\n"); + rc = -ENOMEM; + goto hw_queues_destroy; + } + } + + for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) { + rc = hl_cq_init(hdev, &hdev->completion_queue[i], + hdev->asic_funcs->get_queue_id_for_cq(hdev, i)); + if (rc) { + dev_err(hdev->dev, + "failed to initialize completion queue\n"); + goto cq_fini; + } + hdev->completion_queue[i].cq_idx = i; + } + + hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs, + sizeof(struct hl_cs *), GFP_KERNEL); + if (!hdev->shadow_cs_queue) { + rc = -ENOMEM; + goto cq_fini; + } + + /* + * Initialize the event queue. Must be done before hw_init, + * because there the address of the event queue is being + * passed as argument to request_irq + */ + rc = hl_eq_init(hdev, &hdev->event_queue); + if (rc) { + dev_err(hdev->dev, "failed to initialize event queue\n"); + goto free_shadow_cs_queue; + } + + /* MMU S/W must be initialized before kernel context is created */ + rc = hl_mmu_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n"); + goto eq_fini; + } + + /* Allocate the kernel context */ + hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL); + if (!hdev->kernel_ctx) { + rc = -ENOMEM; + goto mmu_fini; + } + + hdev->is_compute_ctx_active = false; + + hdev->asic_funcs->state_dump_init(hdev); + + hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL; + hl_debugfs_add_device(hdev); + + /* debugfs nodes are created in hl_ctx_init so it must be called after + * hl_debugfs_add_device. + */ + rc = hl_ctx_init(hdev, hdev->kernel_ctx, true); + if (rc) { + dev_err(hdev->dev, "failed to initialize kernel context\n"); + kfree(hdev->kernel_ctx); + goto remove_device_from_debugfs; + } + + rc = hl_cb_pool_init(hdev); + if (rc) { + dev_err(hdev->dev, "failed to initialize CB pool\n"); + goto release_ctx; + } + + rc = hl_dec_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to initialize the decoder module\n"); + goto cb_pool_fini; + } + + /* + * From this point, override rc (=0) in case of an error to allow + * debugging (by adding char devices and create sysfs nodes as part of + * the error flow). + */ + add_cdev_sysfs_on_err = true; + + /* Device is now enabled as part of the initialization requires + * communication with the device firmware to get information that + * is required for the initialization itself + */ + hdev->disabled = false; + + rc = hdev->asic_funcs->hw_init(hdev); + if (rc) { + dev_err(hdev->dev, "failed to initialize the H/W\n"); + rc = 0; + goto out_disabled; + } + + /* Check that the communication with the device is working */ + rc = hdev->asic_funcs->test_queues(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to detect if device is alive\n"); + rc = 0; + goto out_disabled; + } + + rc = device_late_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed late initialization\n"); + rc = 0; + goto out_disabled; + } + + dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n", + hdev->asic_name, + hdev->asic_prop.dram_size / SZ_1G); + + rc = hl_vm_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to initialize memory module\n"); + rc = 0; + goto out_disabled; + } + + /* + * Expose devices and sysfs nodes to user. + * From here there is no need to add char devices and create sysfs nodes + * in case of an error. + */ + add_cdev_sysfs_on_err = false; + rc = device_cdev_sysfs_add(hdev); + if (rc) { + dev_err(hdev->dev, + "Failed to add char devices and sysfs nodes\n"); + rc = 0; + goto out_disabled; + } + + /* Need to call this again because the max power might change, + * depending on card type for certain ASICs + */ + if (hdev->asic_prop.set_max_power_on_device_init && + !hdev->asic_prop.fw_security_enabled) + hl_fw_set_max_power(hdev); + + /* + * hl_hwmon_init() must be called after device_late_init(), because only + * there we get the information from the device about which + * hwmon-related sensors the device supports. + * Furthermore, it must be done after adding the device to the system. + */ + rc = hl_hwmon_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to initialize hwmon\n"); + rc = 0; + goto out_disabled; + } + + dev_notice(hdev->dev, + "Successfully added device %s to habanalabs driver\n", + dev_name(&(hdev)->pdev->dev)); + + hdev->init_done = true; + + /* After initialization is done, we are ready to receive events from + * the F/W. We can't do it before because we will ignore events and if + * those events are fatal, we won't know about it and the device will + * be operational although it shouldn't be + */ + hdev->asic_funcs->enable_events_from_fw(hdev); + + return 0; + +cb_pool_fini: + hl_cb_pool_fini(hdev); +release_ctx: + if (hl_ctx_put(hdev->kernel_ctx) != 1) + dev_err(hdev->dev, + "kernel ctx is still alive on initialization failure\n"); +remove_device_from_debugfs: + hl_debugfs_remove_device(hdev); +mmu_fini: + hl_mmu_fini(hdev); +eq_fini: + hl_eq_fini(hdev, &hdev->event_queue); +free_shadow_cs_queue: + kfree(hdev->shadow_cs_queue); +cq_fini: + for (i = 0 ; i < cq_ready_cnt ; i++) + hl_cq_fini(hdev, &hdev->completion_queue[i]); + kfree(hdev->completion_queue); +hw_queues_destroy: + hl_hw_queues_destroy(hdev); +sw_fini: + hdev->asic_funcs->sw_fini(hdev); +free_usr_intr_mem: + kfree(hdev->user_interrupt); +early_fini: + device_early_fini(hdev); +free_dev_ctrl: + put_device(hdev->dev_ctrl); +free_dev: + put_device(hdev->dev); +out_disabled: + hdev->disabled = true; + if (add_cdev_sysfs_on_err) + device_cdev_sysfs_add(hdev); + if (hdev->pdev) + dev_err(&hdev->pdev->dev, + "Failed to initialize hl%d. Device %s is NOT usable !\n", + hdev->cdev_idx, dev_name(&(hdev)->pdev->dev)); + else + pr_err("Failed to initialize hl%d. Device %s is NOT usable !\n", + hdev->cdev_idx, dev_name(&(hdev)->pdev->dev)); + + return rc; +} + +/* + * hl_device_fini - main tear-down function for habanalabs device + * + * @hdev: pointer to habanalabs device structure + * + * Destroy the device, call ASIC fini functions and release the id + */ +void hl_device_fini(struct hl_device *hdev) +{ + bool device_in_reset; + ktime_t timeout; + u64 reset_sec; + int i, rc; + + dev_info(hdev->dev, "Removing device\n"); + + hdev->device_fini_pending = 1; + flush_delayed_work(&hdev->device_reset_work.reset_work); + + if (hdev->pldm) + reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT; + else + reset_sec = HL_HARD_RESET_MAX_TIMEOUT; + + /* + * This function is competing with the reset function, so try to + * take the reset atomic and if we are already in middle of reset, + * wait until reset function is finished. Reset function is designed + * to always finish. However, in Gaudi, because of all the network + * ports, the hard reset could take between 10-30 seconds + */ + + timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000); + + spin_lock(&hdev->reset_info.lock); + device_in_reset = !!hdev->reset_info.in_reset; + if (!device_in_reset) + hdev->reset_info.in_reset = 1; + spin_unlock(&hdev->reset_info.lock); + + while (device_in_reset) { + usleep_range(50, 200); + + spin_lock(&hdev->reset_info.lock); + device_in_reset = !!hdev->reset_info.in_reset; + if (!device_in_reset) + hdev->reset_info.in_reset = 1; + spin_unlock(&hdev->reset_info.lock); + + if (ktime_compare(ktime_get(), timeout) > 0) { + dev_crit(hdev->dev, + "%s Failed to remove device because reset function did not finish\n", + dev_name(&(hdev)->pdev->dev)); + return; + } + } + + /* Disable PCI access from device F/W so it won't send us additional + * interrupts. We disable MSI/MSI-X at the halt_engines function and we + * can't have the F/W sending us interrupts after that. We need to + * disable the access here because if the device is marked disable, the + * message won't be send. Also, in case of heartbeat, the device CPU is + * marked as disable so this message won't be sent + */ + hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); + + /* Mark device as disabled */ + hdev->disabled = true; + + take_release_locks(hdev); + + hdev->reset_info.hard_reset_pending = true; + + hl_hwmon_fini(hdev); + + cleanup_resources(hdev, true, false, false); + + /* Kill processes here after CS rollback. This is because the process + * can't really exit until all its CSs are done, which is what we + * do in cs rollback + */ + dev_info(hdev->dev, + "Waiting for all processes to exit (timeout of %u seconds)", + HL_PENDING_RESET_LONG_SEC); + + rc = device_kill_open_processes(hdev, HL_PENDING_RESET_LONG_SEC, false); + if (rc) { + dev_crit(hdev->dev, "Failed to kill all open processes\n"); + device_disable_open_processes(hdev, false); + } + + rc = device_kill_open_processes(hdev, 0, true); + if (rc) { + dev_crit(hdev->dev, "Failed to kill all control device open processes\n"); + device_disable_open_processes(hdev, true); + } + + hl_cb_pool_fini(hdev); + + /* Reset the H/W. It will be in idle state after this returns */ + hdev->asic_funcs->hw_fini(hdev, true, false); + + hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE; + + /* Release kernel context */ + if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1)) + dev_err(hdev->dev, "kernel ctx is still alive\n"); + + hl_debugfs_remove_device(hdev); + + hl_dec_fini(hdev); + + hl_vm_fini(hdev); + + hl_mmu_fini(hdev); + + hl_eq_fini(hdev, &hdev->event_queue); + + kfree(hdev->shadow_cs_queue); + + for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) + hl_cq_fini(hdev, &hdev->completion_queue[i]); + kfree(hdev->completion_queue); + kfree(hdev->user_interrupt); + + hl_hw_queues_destroy(hdev); + + /* Call ASIC S/W finalize function */ + hdev->asic_funcs->sw_fini(hdev); + + device_early_fini(hdev); + + /* Hide devices and sysfs nodes from user */ + device_cdev_sysfs_del(hdev); + + pr_info("removed device successfully\n"); +} + +/* + * MMIO register access helper functions. + */ + +/* + * hl_rreg - Read an MMIO register + * + * @hdev: pointer to habanalabs device structure + * @reg: MMIO register offset (in bytes) + * + * Returns the value of the MMIO register we are asked to read + * + */ +inline u32 hl_rreg(struct hl_device *hdev, u32 reg) +{ + return readl(hdev->rmmio + reg); +} + +/* + * hl_wreg - Write to an MMIO register + * + * @hdev: pointer to habanalabs device structure + * @reg: MMIO register offset (in bytes) + * @val: 32-bit value + * + * Writes the 32-bit value into the MMIO register + * + */ +inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val) +{ + writel(val, hdev->rmmio + reg); +} |