Adding upstream version 6.1.76.upstream/6.1.76

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 2258 insertions, 0 deletions

diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c
new file mode 100644
index 000000000..9ee1b6abd
--- /dev/null
+++ b/drivers/misc/habanalabs/common/device.c
@@ -0,0 +1,2258 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2022 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#define pr_fmt(fmt)			"habanalabs: " fmt
+
+#include <uapi/misc/habanalabs.h>
+#include "habanalabs.h"
+
+#include <linux/pci.h>
+#include <linux/hwmon.h>
+
+#include <trace/events/habanalabs.h>
+
+#define HL_RESET_DELAY_USEC		10000	/* 10ms */
+
+enum dma_alloc_type {
+	DMA_ALLOC_COHERENT,
+	DMA_ALLOC_CPU_ACCESSIBLE,
+	DMA_ALLOC_POOL,
+};
+
+#define MEM_SCRUB_DEFAULT_VAL 0x1122334455667788
+
+/*
+ * hl_set_dram_bar- sets the bar to allow later access to address
+ *
+ * @hdev: pointer to habanalabs device structure.
+ * @addr: the address the caller wants to access.
+ * @region: the PCI region.
+ *
+ * @return: the old BAR base address on success, U64_MAX for failure.
+ *	    The caller should set it back to the old address after use.
+ *
+ * In case the bar space does not cover the whole address space,
+ * the bar base address should be set to allow access to a given address.
+ * This function can be called also if the bar doesn't need to be set,
+ * in that case it just won't change the base.
+ */
+static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	u64 bar_base_addr, old_base;
+
+	if (is_power_of_2(prop->dram_pci_bar_size))
+		bar_base_addr = addr & ~(prop->dram_pci_bar_size - 0x1ull);
+	else
+		bar_base_addr = DIV_ROUND_DOWN_ULL(addr, prop->dram_pci_bar_size) *
+				prop->dram_pci_bar_size;
+
+	old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr);
+
+	/* in case of success we need to update the new BAR base */
+	if (old_base != U64_MAX)
+		region->region_base = bar_base_addr;
+
+	return old_base;
+}
+
+static int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val,
+	enum debugfs_access_type acc_type, enum pci_region region_type)
+{
+	struct pci_mem_region *region = &hdev->pci_mem_region[region_type];
+	void __iomem *acc_addr;
+	u64 old_base = 0, rc;
+
+	if (region_type == PCI_REGION_DRAM) {
+		old_base = hl_set_dram_bar(hdev, addr, region);
+		if (old_base == U64_MAX)
+			return -EIO;
+	}
+
+	acc_addr = hdev->pcie_bar[region->bar_id] + addr - region->region_base +
+			region->offset_in_bar;
+	switch (acc_type) {
+	case DEBUGFS_READ8:
+		*val = readb(acc_addr);
+		break;
+	case DEBUGFS_WRITE8:
+		writeb(*val, acc_addr);
+		break;
+	case DEBUGFS_READ32:
+		*val = readl(acc_addr);
+		break;
+	case DEBUGFS_WRITE32:
+		writel(*val, acc_addr);
+		break;
+	case DEBUGFS_READ64:
+		*val = readq(acc_addr);
+		break;
+	case DEBUGFS_WRITE64:
+		writeq(*val, acc_addr);
+		break;
+	}
+
+	if (region_type == PCI_REGION_DRAM) {
+		rc = hl_set_dram_bar(hdev, old_base, region);
+		if (rc == U64_MAX)
+			return -EIO;
+	}
+
+	return 0;
+}
+
+static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
+					gfp_t flag, enum dma_alloc_type alloc_type,
+					const char *caller)
+{
+	void *ptr = NULL;
+
+	switch (alloc_type) {
+	case DMA_ALLOC_COHERENT:
+		ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag);
+		break;
+	case DMA_ALLOC_CPU_ACCESSIBLE:
+		ptr = hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
+		break;
+	case DMA_ALLOC_POOL:
+		ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle);
+		break;
+	}
+
+	if (trace_habanalabs_dma_alloc_enabled() && !ZERO_OR_NULL_PTR(ptr))
+		trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size,
+						caller);
+
+	return ptr;
+}
+
+static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr,
+					dma_addr_t dma_handle, enum dma_alloc_type alloc_type,
+					const char *caller)
+{
+	switch (alloc_type) {
+	case DMA_ALLOC_COHERENT:
+		hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle);
+		break;
+	case DMA_ALLOC_CPU_ACCESSIBLE:
+		hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, cpu_addr);
+		break;
+	case DMA_ALLOC_POOL:
+		hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle);
+		break;
+	}
+
+	trace_habanalabs_dma_free(hdev->dev, (u64) (uintptr_t) cpu_addr, dma_handle, size, caller);
+}
+
+void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
+					gfp_t flag, const char *caller)
+{
+	return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller);
+}
+
+void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr,
+					dma_addr_t dma_handle, const char *caller)
+{
+	hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller);
+}
+
+void *hl_cpu_accessible_dma_pool_alloc_caller(struct hl_device *hdev, size_t size,
+						dma_addr_t *dma_handle, const char *caller)
+{
+	return hl_dma_alloc_common(hdev, size, dma_handle, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller);
+}
+
+void hl_cpu_accessible_dma_pool_free_caller(struct hl_device *hdev, size_t size, void *vaddr,
+						const char *caller)
+{
+	hl_asic_dma_free_common(hdev, size, vaddr, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller);
+}
+
+void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags,
+					dma_addr_t *dma_handle, const char *caller)
+{
+	return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller);
+}
+
+void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr,
+					const char *caller)
+{
+	hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller);
+}
+
+int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct scatterlist *sg;
+	int rc, i;
+
+	rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0);
+	if (rc)
+		return rc;
+
+	/* Shift to the device's base physical address of host memory if necessary */
+	if (prop->device_dma_offset_for_host_access)
+		for_each_sgtable_dma_sg(sgt, sg, i)
+			sg->dma_address += prop->device_dma_offset_for_host_access;
+
+	return 0;
+}
+
+void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct scatterlist *sg;
+	int i;
+
+	/* Cancel the device's base physical address of host memory if necessary */
+	if (prop->device_dma_offset_for_host_access)
+		for_each_sgtable_dma_sg(sgt, sg, i)
+			sg->dma_address -= prop->device_dma_offset_for_host_access;
+
+	dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0);
+}
+
+/*
+ * hl_access_cfg_region - access the config region
+ *
+ * @hdev: pointer to habanalabs device structure
+ * @addr: the address to access
+ * @val: the value to write from or read to
+ * @acc_type: the type of access (read/write 64/32)
+ */
+int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val,
+	enum debugfs_access_type acc_type)
+{
+	struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG];
+	u32 val_h, val_l;
+
+	if (!IS_ALIGNED(addr, sizeof(u32))) {
+		dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32));
+		return -EINVAL;
+	}
+
+	switch (acc_type) {
+	case DEBUGFS_READ32:
+		*val = RREG32(addr - cfg_region->region_base);
+		break;
+	case DEBUGFS_WRITE32:
+		WREG32(addr - cfg_region->region_base, *val);
+		break;
+	case DEBUGFS_READ64:
+		val_l = RREG32(addr - cfg_region->region_base);
+		val_h = RREG32(addr + sizeof(u32) - cfg_region->region_base);
+
+		*val = (((u64) val_h) << 32) | val_l;
+		break;
+	case DEBUGFS_WRITE64:
+		WREG32(addr - cfg_region->region_base, lower_32_bits(*val));
+		WREG32(addr + sizeof(u32) - cfg_region->region_base, upper_32_bits(*val));
+		break;
+	default:
+		dev_err(hdev->dev, "access type %d is not supported\n", acc_type);
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+/*
+ * hl_access_dev_mem - access device memory
+ *
+ * @hdev: pointer to habanalabs device structure
+ * @region_type: the type of the region the address belongs to
+ * @addr: the address to access
+ * @val: the value to write from or read to
+ * @acc_type: the type of access (r/w, 32/64)
+ */
+int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type,
+			u64 addr, u64 *val, enum debugfs_access_type acc_type)
+{
+	switch (region_type) {
+	case PCI_REGION_CFG:
+		return hl_access_cfg_region(hdev, addr, val, acc_type);
+	case PCI_REGION_SRAM:
+	case PCI_REGION_DRAM:
+		return hl_access_sram_dram_region(hdev, addr, val, acc_type,
+			region_type);
+	default:
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...)
+{
+	va_list args;
+	int str_size;
+
+	va_start(args, fmt);
+	/* Calculate formatted string length. Assuming each string is null terminated, hence
+	 * increment result by 1
+	 */
+	str_size = vsnprintf(NULL, 0, fmt, args) + 1;
+	va_end(args);
+
+	if ((e->actual_size + str_size) < e->allocated_buf_size) {
+		va_start(args, fmt);
+		vsnprintf(e->buf + e->actual_size, str_size, fmt, args);
+		va_end(args);
+	}
+
+	/* Need to update the size even when not updating destination buffer to get the exact size
+	 * of all input strings
+	 */
+	e->actual_size += str_size;
+}
+
+enum hl_device_status hl_device_status(struct hl_device *hdev)
+{
+	enum hl_device_status status;
+
+	if (hdev->reset_info.in_reset) {
+		if (hdev->reset_info.in_compute_reset)
+			status = HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE;
+		else
+			status = HL_DEVICE_STATUS_IN_RESET;
+	} else if (hdev->reset_info.needs_reset) {
+		status = HL_DEVICE_STATUS_NEEDS_RESET;
+	} else if (hdev->disabled) {
+		status = HL_DEVICE_STATUS_MALFUNCTION;
+	} else if (!hdev->init_done) {
+		status = HL_DEVICE_STATUS_IN_DEVICE_CREATION;
+	} else {
+		status = HL_DEVICE_STATUS_OPERATIONAL;
+	}
+
+	return status;
+}
+
+bool hl_device_operational(struct hl_device *hdev,
+		enum hl_device_status *status)
+{
+	enum hl_device_status current_status;
+
+	current_status = hl_device_status(hdev);
+	if (status)
+		*status = current_status;
+
+	switch (current_status) {
+	case HL_DEVICE_STATUS_IN_RESET:
+	case HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE:
+	case HL_DEVICE_STATUS_MALFUNCTION:
+	case HL_DEVICE_STATUS_NEEDS_RESET:
+		return false;
+	case HL_DEVICE_STATUS_OPERATIONAL:
+	case HL_DEVICE_STATUS_IN_DEVICE_CREATION:
+	default:
+		return true;
+	}
+}
+
+static void hpriv_release(struct kref *ref)
+{
+	u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
+	bool device_is_idle = true;
+	struct hl_fpriv *hpriv;
+	struct hl_device *hdev;
+
+	hpriv = container_of(ref, struct hl_fpriv, refcount);
+
+	hdev = hpriv->hdev;
+
+	hdev->asic_funcs->send_device_activity(hdev, false);
+
+	put_pid(hpriv->taskpid);
+
+	hl_debugfs_remove_file(hpriv);
+
+	mutex_destroy(&hpriv->ctx_lock);
+	mutex_destroy(&hpriv->restore_phase_mutex);
+
+	if ((!hdev->pldm) && (hdev->pdev) &&
+			(!hdev->asic_funcs->is_device_idle(hdev,
+				idle_mask,
+				HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) {
+		dev_err(hdev->dev,
+			"device not idle after user context is closed (0x%llx_%llx)\n",
+			idle_mask[1], idle_mask[0]);
+
+		device_is_idle = false;
+	}
+
+	/* We need to remove the user from the list to make sure the reset process won't
+	 * try to kill the user process. Because, if we got here, it means there are no
+	 * more driver/device resources that the user process is occupying so there is
+	 * no need to kill it
+	 *
+	 * However, we can't set the compute_ctx to NULL at this stage. This is to prevent
+	 * a race between the release and opening the device again. We don't want to let
+	 * a user open the device while there a reset is about to happen.
+	 */
+	mutex_lock(&hdev->fpriv_list_lock);
+	list_del(&hpriv->dev_node);
+	mutex_unlock(&hdev->fpriv_list_lock);
+
+	if (!device_is_idle || hdev->reset_upon_device_release) {
+		hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE);
+	} else {
+		int rc = hdev->asic_funcs->scrub_device_mem(hdev);
+
+		if (rc)
+			dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc);
+	}
+
+	/* Now we can mark the compute_ctx as not active. Even if a reset is running in a different
+	 * thread, we don't care because the in_reset is marked so if a user will try to open
+	 * the device it will fail on that, even if compute_ctx is false.
+	 */
+	mutex_lock(&hdev->fpriv_list_lock);
+	hdev->is_compute_ctx_active = false;
+	mutex_unlock(&hdev->fpriv_list_lock);
+
+	hdev->compute_ctx_in_release = 0;
+
+	/* release the eventfd */
+	if (hpriv->notifier_event.eventfd)
+		eventfd_ctx_put(hpriv->notifier_event.eventfd);
+
+	mutex_destroy(&hpriv->notifier_event.lock);
+
+	kfree(hpriv);
+}
+
+void hl_hpriv_get(struct hl_fpriv *hpriv)
+{
+	kref_get(&hpriv->refcount);
+}
+
+int hl_hpriv_put(struct hl_fpriv *hpriv)
+{
+	return kref_put(&hpriv->refcount, hpriv_release);
+}
+
+/*
+ * hl_device_release - release function for habanalabs device
+ *
+ * @inode: pointer to inode structure
+ * @filp: pointer to file structure
+ *
+ * Called when process closes an habanalabs device
+ */
+static int hl_device_release(struct inode *inode, struct file *filp)
+{
+	struct hl_fpriv *hpriv = filp->private_data;
+	struct hl_device *hdev = hpriv->hdev;
+
+	filp->private_data = NULL;
+
+	if (!hdev) {
+		pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n");
+		put_pid(hpriv->taskpid);
+		return 0;
+	}
+
+	/* Each pending user interrupt holds the user's context, hence we
+	 * must release them all before calling hl_ctx_mgr_fini().
+	 */
+	hl_release_pending_user_interrupts(hpriv->hdev);
+
+	hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
+	hl_mem_mgr_fini(&hpriv->mem_mgr);
+
+	hdev->compute_ctx_in_release = 1;
+
+	if (!hl_hpriv_put(hpriv))
+		dev_notice(hdev->dev,
+			"User process closed FD but device still in use\n");
+
+	hdev->last_open_session_duration_jif =
+		jiffies - hdev->last_successful_open_jif;
+
+	return 0;
+}
+
+static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
+{
+	struct hl_fpriv *hpriv = filp->private_data;
+	struct hl_device *hdev = hpriv->hdev;
+
+	filp->private_data = NULL;
+
+	if (!hdev) {
+		pr_err("Closing FD after device was removed\n");
+		goto out;
+	}
+
+	mutex_lock(&hdev->fpriv_ctrl_list_lock);
+	list_del(&hpriv->dev_node);
+	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
+out:
+	/* release the eventfd */
+	if (hpriv->notifier_event.eventfd)
+		eventfd_ctx_put(hpriv->notifier_event.eventfd);
+
+	mutex_destroy(&hpriv->notifier_event.lock);
+	put_pid(hpriv->taskpid);
+
+	kfree(hpriv);
+
+	return 0;
+}
+
+/*
+ * hl_mmap - mmap function for habanalabs device
+ *
+ * @*filp: pointer to file structure
+ * @*vma: pointer to vm_area_struct of the process
+ *
+ * Called when process does an mmap on habanalabs device. Call the relevant mmap
+ * function at the end of the common code.
+ */
+static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct hl_fpriv *hpriv = filp->private_data;
+	struct hl_device *hdev = hpriv->hdev;
+	unsigned long vm_pgoff;
+
+	if (!hdev) {
+		pr_err_ratelimited("Trying to mmap after device was removed! Please close FD\n");
+		return -ENODEV;
+	}
+
+	vm_pgoff = vma->vm_pgoff;
+
+	switch (vm_pgoff & HL_MMAP_TYPE_MASK) {
+	case HL_MMAP_TYPE_BLOCK:
+		vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff);
+		return hl_hw_block_mmap(hpriv, vma);
+
+	case HL_MMAP_TYPE_CB:
+	case HL_MMAP_TYPE_TS_BUFF:
+		return hl_mem_mgr_mmap(&hpriv->mem_mgr, vma, NULL);
+	}
+	return -EINVAL;
+}
+
+static const struct file_operations hl_ops = {
+	.owner = THIS_MODULE,
+	.open = hl_device_open,
+	.release = hl_device_release,
+	.mmap = hl_mmap,
+	.unlocked_ioctl = hl_ioctl,
+	.compat_ioctl = hl_ioctl
+};
+
+static const struct file_operations hl_ctrl_ops = {
+	.owner = THIS_MODULE,
+	.open = hl_device_open_ctrl,
+	.release = hl_device_release_ctrl,
+	.unlocked_ioctl = hl_ioctl_control,
+	.compat_ioctl = hl_ioctl_control
+};
+
+static void device_release_func(struct device *dev)
+{
+	kfree(dev);
+}
+
+/*
+ * device_init_cdev - Initialize cdev and device for habanalabs device
+ *
+ * @hdev: pointer to habanalabs device structure
+ * @hclass: pointer to the class object of the device
+ * @minor: minor number of the specific device
+ * @fpos: file operations to install for this device
+ * @name: name of the device as it will appear in the filesystem
+ * @cdev: pointer to the char device object that will be initialized
+ * @dev: pointer to the device object that will be initialized
+ *
+ * Initialize a cdev and a Linux device for habanalabs's device.
+ */
+static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
+				int minor, const struct file_operations *fops,
+				char *name, struct cdev *cdev,
+				struct device **dev)
+{
+	cdev_init(cdev, fops);
+	cdev->owner = THIS_MODULE;
+
+	*dev = kzalloc(sizeof(**dev), GFP_KERNEL);
+	if (!*dev)
+		return -ENOMEM;
+
+	device_initialize(*dev);
+	(*dev)->devt = MKDEV(hdev->major, minor);
+	(*dev)->class = hclass;
+	(*dev)->release = device_release_func;
+	dev_set_drvdata(*dev, hdev);
+	dev_set_name(*dev, "%s", name);
+
+	return 0;
+}
+
+static int device_cdev_sysfs_add(struct hl_device *hdev)
+{
+	int rc;
+
+	rc = cdev_device_add(&hdev->cdev, hdev->dev);
+	if (rc) {
+		dev_err(hdev->dev,
+			"failed to add a char device to the system\n");
+		return rc;
+	}
+
+	rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
+	if (rc) {
+		dev_err(hdev->dev,
+			"failed to add a control char device to the system\n");
+		goto delete_cdev_device;
+	}
+
+	/* hl_sysfs_init() must be done after adding the device to the system */
+	rc = hl_sysfs_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize sysfs\n");
+		goto delete_ctrl_cdev_device;
+	}
+
+	hdev->cdev_sysfs_created = true;
+
+	return 0;
+
+delete_ctrl_cdev_device:
+	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
+delete_cdev_device:
+	cdev_device_del(&hdev->cdev, hdev->dev);
+	return rc;
+}
+
+static void device_cdev_sysfs_del(struct hl_device *hdev)
+{
+	if (!hdev->cdev_sysfs_created)
+		goto put_devices;
+
+	hl_sysfs_fini(hdev);
+	cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
+	cdev_device_del(&hdev->cdev, hdev->dev);
+
+put_devices:
+	put_device(hdev->dev);
+	put_device(hdev->dev_ctrl);
+}
+
+static void device_hard_reset_pending(struct work_struct *work)
+{
+	struct hl_device_reset_work *device_reset_work =
+		container_of(work, struct hl_device_reset_work, reset_work.work);
+	struct hl_device *hdev = device_reset_work->hdev;
+	u32 flags;
+	int rc;
+
+	flags = device_reset_work->flags | HL_DRV_RESET_FROM_RESET_THR;
+
+	rc = hl_device_reset(hdev, flags);
+	if ((rc == -EBUSY) && !hdev->device_fini_pending) {
+		dev_info(hdev->dev,
+			"Could not reset device. will try again in %u seconds",
+			HL_PENDING_RESET_PER_SEC);
+
+		queue_delayed_work(device_reset_work->wq,
+			&device_reset_work->reset_work,
+			msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000));
+	}
+}
+
+/*
+ * device_early_init - do some early initialization for the habanalabs device
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ * Install the relevant function pointers and call the early_init function,
+ * if such a function exists
+ */
+static int device_early_init(struct hl_device *hdev)
+{
+	int i, rc;
+	char workq_name[32];
+
+	switch (hdev->asic_type) {
+	case ASIC_GOYA:
+		goya_set_asic_funcs(hdev);
+		strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
+		break;
+	case ASIC_GAUDI:
+		gaudi_set_asic_funcs(hdev);
+		strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name));
+		break;
+	case ASIC_GAUDI_SEC:
+		gaudi_set_asic_funcs(hdev);
+		strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name));
+		break;
+	case ASIC_GAUDI2:
+		gaudi2_set_asic_funcs(hdev);
+		strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name));
+		break;
+	case ASIC_GAUDI2_SEC:
+		gaudi2_set_asic_funcs(hdev);
+		strscpy(hdev->asic_name, "GAUDI2 SEC", sizeof(hdev->asic_name));
+		break;
+	default:
+		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
+			hdev->asic_type);
+		return -EINVAL;
+	}
+
+	rc = hdev->asic_funcs->early_init(hdev);
+	if (rc)
+		return rc;
+
+	rc = hl_asid_init(hdev);
+	if (rc)
+		goto early_fini;
+
+	if (hdev->asic_prop.completion_queues_count) {
+		hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
+				sizeof(struct workqueue_struct *),
+				GFP_KERNEL);
+		if (!hdev->cq_wq) {
+			rc = -ENOMEM;
+			goto asid_fini;
+		}
+	}
+
+	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
+		snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i);
+		hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
+		if (hdev->cq_wq[i] == NULL) {
+			dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
+			rc = -ENOMEM;
+			goto free_cq_wq;
+		}
+	}
+
+	hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
+	if (hdev->eq_wq == NULL) {
+		dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
+		rc = -ENOMEM;
+		goto free_cq_wq;
+	}
+
+	hdev->cs_cmplt_wq = alloc_workqueue("hl-cs-completions", WQ_UNBOUND, 0);
+	if (!hdev->cs_cmplt_wq) {
+		dev_err(hdev->dev,
+			"Failed to allocate CS completions workqueue\n");
+		rc = -ENOMEM;
+		goto free_eq_wq;
+	}
+
+	hdev->ts_free_obj_wq = alloc_workqueue("hl-ts-free-obj", WQ_UNBOUND, 0);
+	if (!hdev->ts_free_obj_wq) {
+		dev_err(hdev->dev,
+			"Failed to allocate Timestamp registration free workqueue\n");
+		rc = -ENOMEM;
+		goto free_cs_cmplt_wq;
+	}
+
+	hdev->pf_wq = alloc_workqueue("hl-prefetch", WQ_UNBOUND, 0);
+	if (!hdev->pf_wq) {
+		dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n");
+		rc = -ENOMEM;
+		goto free_ts_free_wq;
+	}
+
+	hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
+					GFP_KERNEL);
+	if (!hdev->hl_chip_info) {
+		rc = -ENOMEM;
+		goto free_pf_wq;
+	}
+
+	rc = hl_mmu_if_set_funcs(hdev);
+	if (rc)
+		goto free_chip_info;
+
+	hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr);
+
+	hdev->device_reset_work.wq =
+			create_singlethread_workqueue("hl_device_reset");
+	if (!hdev->device_reset_work.wq) {
+		rc = -ENOMEM;
+		dev_err(hdev->dev, "Failed to create device reset WQ\n");
+		goto free_cb_mgr;
+	}
+
+	INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work,
+			device_hard_reset_pending);
+	hdev->device_reset_work.hdev = hdev;
+	hdev->device_fini_pending = 0;
+
+	mutex_init(&hdev->send_cpu_message_lock);
+	mutex_init(&hdev->debug_lock);
+	INIT_LIST_HEAD(&hdev->cs_mirror_list);
+	spin_lock_init(&hdev->cs_mirror_lock);
+	spin_lock_init(&hdev->reset_info.lock);
+	INIT_LIST_HEAD(&hdev->fpriv_list);
+	INIT_LIST_HEAD(&hdev->fpriv_ctrl_list);
+	mutex_init(&hdev->fpriv_list_lock);
+	mutex_init(&hdev->fpriv_ctrl_list_lock);
+	mutex_init(&hdev->clk_throttling.lock);
+
+	return 0;
+
+free_cb_mgr:
+	hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
+free_chip_info:
+	kfree(hdev->hl_chip_info);
+free_pf_wq:
+	destroy_workqueue(hdev->pf_wq);
+free_ts_free_wq:
+	destroy_workqueue(hdev->ts_free_obj_wq);
+free_cs_cmplt_wq:
+	destroy_workqueue(hdev->cs_cmplt_wq);
+free_eq_wq:
+	destroy_workqueue(hdev->eq_wq);
+free_cq_wq:
+	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
+		if (hdev->cq_wq[i])
+			destroy_workqueue(hdev->cq_wq[i]);
+	kfree(hdev->cq_wq);
+asid_fini:
+	hl_asid_fini(hdev);
+early_fini:
+	if (hdev->asic_funcs->early_fini)
+		hdev->asic_funcs->early_fini(hdev);
+
+	return rc;
+}
+
+/*
+ * device_early_fini - finalize all that was done in device_early_init
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ */
+static void device_early_fini(struct hl_device *hdev)
+{
+	int i;
+
+	mutex_destroy(&hdev->debug_lock);
+	mutex_destroy(&hdev->send_cpu_message_lock);
+
+	mutex_destroy(&hdev->fpriv_list_lock);
+	mutex_destroy(&hdev->fpriv_ctrl_list_lock);
+
+	mutex_destroy(&hdev->clk_throttling.lock);
+
+	hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
+
+	kfree(hdev->hl_chip_info);
+
+	destroy_workqueue(hdev->pf_wq);
+	destroy_workqueue(hdev->ts_free_obj_wq);
+	destroy_workqueue(hdev->cs_cmplt_wq);
+	destroy_workqueue(hdev->eq_wq);
+	destroy_workqueue(hdev->device_reset_work.wq);
+
+	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
+		destroy_workqueue(hdev->cq_wq[i]);
+	kfree(hdev->cq_wq);
+
+	hl_asid_fini(hdev);
+
+	if (hdev->asic_funcs->early_fini)
+		hdev->asic_funcs->early_fini(hdev);
+}
+
+static bool is_pci_link_healthy(struct hl_device *hdev)
+{
+	u16 vendor_id;
+
+	if (!hdev->pdev)
+		return false;
+
+	pci_read_config_word(hdev->pdev, PCI_VENDOR_ID, &vendor_id);
+
+	return (vendor_id == PCI_VENDOR_ID_HABANALABS);
+}
+
+static void hl_device_heartbeat(struct work_struct *work)
+{
+	struct hl_device *hdev = container_of(work, struct hl_device,
+						work_heartbeat.work);
+
+	if (!hl_device_operational(hdev, NULL))
+		goto reschedule;
+
+	if (!hdev->asic_funcs->send_heartbeat(hdev))
+		goto reschedule;
+
+	if (hl_device_operational(hdev, NULL))
+		dev_err(hdev->dev, "Device heartbeat failed! PCI link is %s\n",
+			is_pci_link_healthy(hdev) ? "healthy" : "broken");
+
+	hl_device_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT);
+
+	return;
+
+reschedule:
+	/*
+	 * prev_reset_trigger tracks consecutive fatal h/w errors until first
+	 * heartbeat immediately post reset.
+	 * If control reached here, then at least one heartbeat work has been
+	 * scheduled since last reset/init cycle.
+	 * So if the device is not already in reset cycle, reset the flag
+	 * prev_reset_trigger as no reset occurred with HL_DRV_RESET_FW_FATAL_ERR
+	 * status for at least one heartbeat. From this point driver restarts
+	 * tracking future consecutive fatal errors.
+	 */
+	if (!hdev->reset_info.in_reset)
+		hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
+
+	schedule_delayed_work(&hdev->work_heartbeat,
+			usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
+}
+
+/*
+ * device_late_init - do late stuff initialization for the habanalabs device
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ * Do stuff that either needs the device H/W queues to be active or needs
+ * to happen after all the rest of the initialization is finished
+ */
+static int device_late_init(struct hl_device *hdev)
+{
+	int rc;
+
+	if (hdev->asic_funcs->late_init) {
+		rc = hdev->asic_funcs->late_init(hdev);
+		if (rc) {
+			dev_err(hdev->dev,
+				"failed late initialization for the H/W\n");
+			return rc;
+		}
+	}
+
+	hdev->high_pll = hdev->asic_prop.high_pll;
+
+	if (hdev->heartbeat) {
+		INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
+		schedule_delayed_work(&hdev->work_heartbeat,
+				usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
+	}
+
+	hdev->late_init_done = true;
+
+	return 0;
+}
+
+/*
+ * device_late_fini - finalize all that was done in device_late_init
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ */
+static void device_late_fini(struct hl_device *hdev)
+{
+	if (!hdev->late_init_done)
+		return;
+
+	if (hdev->heartbeat)
+		cancel_delayed_work_sync(&hdev->work_heartbeat);
+
+	if (hdev->asic_funcs->late_fini)
+		hdev->asic_funcs->late_fini(hdev);
+
+	hdev->late_init_done = false;
+}
+
+int hl_device_utilization(struct hl_device *hdev, u32 *utilization)
+{
+	u64 max_power, curr_power, dc_power, dividend;
+	int rc;
+
+	max_power = hdev->max_power;
+	dc_power = hdev->asic_prop.dc_power_default;
+	rc = hl_fw_cpucp_power_get(hdev, &curr_power);
+
+	if (rc)
+		return rc;
+
+	curr_power = clamp(curr_power, dc_power, max_power);
+
+	dividend = (curr_power - dc_power) * 100;
+	*utilization = (u32) div_u64(dividend, (max_power - dc_power));
+
+	return 0;
+}
+
+int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable)
+{
+	int rc = 0;
+
+	mutex_lock(&hdev->debug_lock);
+
+	if (!enable) {
+		if (!hdev->in_debug) {
+			dev_err(hdev->dev,
+				"Failed to disable debug mode because device was not in debug mode\n");
+			rc = -EFAULT;
+			goto out;
+		}
+
+		if (!hdev->reset_info.hard_reset_pending)
+			hdev->asic_funcs->halt_coresight(hdev, ctx);
+
+		hdev->in_debug = 0;
+
+		goto out;
+	}
+
+	if (hdev->in_debug) {
+		dev_err(hdev->dev,
+			"Failed to enable debug mode because device is already in debug mode\n");
+		rc = -EFAULT;
+		goto out;
+	}
+
+	hdev->in_debug = 1;
+
+out:
+	mutex_unlock(&hdev->debug_lock);
+
+	return rc;
+}
+
+static void take_release_locks(struct hl_device *hdev)
+{
+	/* Flush anyone that is inside the critical section of enqueue
+	 * jobs to the H/W
+	 */
+	hdev->asic_funcs->hw_queues_lock(hdev);
+	hdev->asic_funcs->hw_queues_unlock(hdev);
+
+	/* Flush processes that are sending message to CPU */
+	mutex_lock(&hdev->send_cpu_message_lock);
+	mutex_unlock(&hdev->send_cpu_message_lock);
+
+	/* Flush anyone that is inside device open */
+	mutex_lock(&hdev->fpriv_list_lock);
+	mutex_unlock(&hdev->fpriv_list_lock);
+	mutex_lock(&hdev->fpriv_ctrl_list_lock);
+	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
+}
+
+static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset,
+				bool skip_wq_flush)
+{
+	if (hard_reset)
+		device_late_fini(hdev);
+
+	/*
+	 * Halt the engines and disable interrupts so we won't get any more
+	 * completions from H/W and we won't have any accesses from the
+	 * H/W to the host machine
+	 */
+	hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset);
+
+	/* Go over all the queues, release all CS and their jobs */
+	hl_cs_rollback_all(hdev, skip_wq_flush);
+
+	/* flush the MMU prefetch workqueue */
+	flush_workqueue(hdev->pf_wq);
+
+	/* Release all pending user interrupts, each pending user interrupt
+	 * holds a reference to user context
+	 */
+	hl_release_pending_user_interrupts(hdev);
+}
+
+/*
+ * hl_device_suspend - initiate device suspend
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ * Puts the hw in the suspend state (all asics).
+ * Returns 0 for success or an error on failure.
+ * Called at driver suspend.
+ */
+int hl_device_suspend(struct hl_device *hdev)
+{
+	int rc;
+
+	pci_save_state(hdev->pdev);
+
+	/* Block future CS/VM/JOB completion operations */
+	spin_lock(&hdev->reset_info.lock);
+	if (hdev->reset_info.in_reset) {
+		spin_unlock(&hdev->reset_info.lock);
+		dev_err(hdev->dev, "Can't suspend while in reset\n");
+		return -EIO;
+	}
+	hdev->reset_info.in_reset = 1;
+	spin_unlock(&hdev->reset_info.lock);
+
+	/* This blocks all other stuff that is not blocked by in_reset */
+	hdev->disabled = true;
+
+	take_release_locks(hdev);
+
+	rc = hdev->asic_funcs->suspend(hdev);
+	if (rc)
+		dev_err(hdev->dev,
+			"Failed to disable PCI access of device CPU\n");
+
+	/* Shut down the device */
+	pci_disable_device(hdev->pdev);
+	pci_set_power_state(hdev->pdev, PCI_D3hot);
+
+	return 0;
+}
+
+/*
+ * hl_device_resume - initiate device resume
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ * Bring the hw back to operating state (all asics).
+ * Returns 0 for success or an error on failure.
+ * Called at driver resume.
+ */
+int hl_device_resume(struct hl_device *hdev)
+{
+	int rc;
+
+	pci_set_power_state(hdev->pdev, PCI_D0);
+	pci_restore_state(hdev->pdev);
+	rc = pci_enable_device_mem(hdev->pdev);
+	if (rc) {
+		dev_err(hdev->dev,
+			"Failed to enable PCI device in resume\n");
+		return rc;
+	}
+
+	pci_set_master(hdev->pdev);
+
+	rc = hdev->asic_funcs->resume(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to resume device after suspend\n");
+		goto disable_device;
+	}
+
+
+	/* 'in_reset' was set to true during suspend, now we must clear it in order
+	 * for hard reset to be performed
+	 */
+	spin_lock(&hdev->reset_info.lock);
+	hdev->reset_info.in_reset = 0;
+	spin_unlock(&hdev->reset_info.lock);
+
+	rc = hl_device_reset(hdev, HL_DRV_RESET_HARD);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to reset device during resume\n");
+		goto disable_device;
+	}
+
+	return 0;
+
+disable_device:
+	pci_clear_master(hdev->pdev);
+	pci_disable_device(hdev->pdev);
+
+	return rc;
+}
+
+static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev)
+{
+	struct task_struct *task = NULL;
+	struct list_head *fd_list;
+	struct hl_fpriv	*hpriv;
+	struct mutex *fd_lock;
+	u32 pending_cnt;
+
+	fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
+	fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
+
+	/* Giving time for user to close FD, and for processes that are inside
+	 * hl_device_open to finish
+	 */
+	if (!list_empty(fd_list))
+		ssleep(1);
+
+	if (timeout) {
+		pending_cnt = timeout;
+	} else {
+		if (hdev->process_kill_trial_cnt) {
+			/* Processes have been already killed */
+			pending_cnt = 1;
+			goto wait_for_processes;
+		} else {
+			/* Wait a small period after process kill */
+			pending_cnt = HL_PENDING_RESET_PER_SEC;
+		}
+	}
+
+	mutex_lock(fd_lock);
+
+	/* This section must be protected because we are dereferencing
+	 * pointers that are freed if the process exits
+	 */
+	list_for_each_entry(hpriv, fd_list, dev_node) {
+		task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
+		if (task) {
+			dev_info(hdev->dev, "Killing user process pid=%d\n",
+				task_pid_nr(task));
+			send_sig(SIGKILL, task, 1);
+			usleep_range(1000, 10000);
+
+			put_task_struct(task);
+		} else {
+			/*
+			 * If we got here, it means that process was killed from outside the driver
+			 * right after it started looping on fd_list and before get_pid_task, thus
+			 * we don't need to kill it.
+			 */
+			dev_dbg(hdev->dev,
+				"Can't get task struct for user process, assuming process was killed from outside the driver\n");
+		}
+	}
+
+	mutex_unlock(fd_lock);
+
+	/*
+	 * We killed the open users, but that doesn't mean they are closed.
+	 * It could be that they are running a long cleanup phase in the driver
+	 * e.g. MMU unmappings, or running other long teardown flow even before
+	 * our cleanup.
+	 * Therefore we need to wait again to make sure they are closed before
+	 * continuing with the reset.
+	 */
+
+wait_for_processes:
+	while ((!list_empty(fd_list)) && (pending_cnt)) {
+		dev_dbg(hdev->dev,
+			"Waiting for all unmap operations to finish before hard reset\n");
+
+		pending_cnt--;
+
+		ssleep(1);
+	}
+
+	/* All processes exited successfully */
+	if (list_empty(fd_list))
+		return 0;
+
+	/* Give up waiting for processes to exit */
+	if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS)
+		return -ETIME;
+
+	hdev->process_kill_trial_cnt++;
+
+	return -EBUSY;
+}
+
+static void device_disable_open_processes(struct hl_device *hdev, bool control_dev)
+{
+	struct list_head *fd_list;
+	struct hl_fpriv *hpriv;
+	struct mutex *fd_lock;
+
+	fd_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
+	fd_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
+
+	mutex_lock(fd_lock);
+	list_for_each_entry(hpriv, fd_list, dev_node)
+		hpriv->hdev = NULL;
+	mutex_unlock(fd_lock);
+}
+
+static void handle_reset_trigger(struct hl_device *hdev, u32 flags)
+{
+	u32 cur_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
+
+	/*
+	 * 'reset cause' is being updated here, because getting here
+	 * means that it's the 1st time and the last time we're here
+	 * ('in_reset' makes sure of it). This makes sure that
+	 * 'reset_cause' will continue holding its 1st recorded reason!
+	 */
+	if (flags & HL_DRV_RESET_HEARTBEAT) {
+		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT;
+		cur_reset_trigger = HL_DRV_RESET_HEARTBEAT;
+	} else if (flags & HL_DRV_RESET_TDR) {
+		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR;
+		cur_reset_trigger = HL_DRV_RESET_TDR;
+	} else if (flags & HL_DRV_RESET_FW_FATAL_ERR) {
+		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
+		cur_reset_trigger = HL_DRV_RESET_FW_FATAL_ERR;
+	} else {
+		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
+	}
+
+	/*
+	 * If reset cause is same twice, then reset_trigger_repeated
+	 * is set and if this reset is due to a fatal FW error
+	 * device is set to an unstable state.
+	 */
+	if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) {
+		hdev->reset_info.prev_reset_trigger = cur_reset_trigger;
+		hdev->reset_info.reset_trigger_repeated = 0;
+	} else {
+		hdev->reset_info.reset_trigger_repeated = 1;
+	}
+
+	/* If reset is due to heartbeat, device CPU is no responsive in
+	 * which case no point sending PCI disable message to it.
+	 *
+	 * If F/W is performing the reset, no need to send it a message to disable
+	 * PCI access
+	 */
+	if ((flags & HL_DRV_RESET_HARD) &&
+			!(flags & (HL_DRV_RESET_HEARTBEAT | HL_DRV_RESET_BYPASS_REQ_TO_FW))) {
+		/* Disable PCI access from device F/W so he won't send
+		 * us additional interrupts. We disable MSI/MSI-X at
+		 * the halt_engines function and we can't have the F/W
+		 * sending us interrupts after that. We need to disable
+		 * the access here because if the device is marked
+		 * disable, the message won't be send. Also, in case
+		 * of heartbeat, the device CPU is marked as disable
+		 * so this message won't be sent
+		 */
+		if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0))
+			dev_warn(hdev->dev,
+				"Failed to disable PCI access by F/W\n");
+	}
+}
+
+/*
+ * hl_device_reset - reset the device
+ *
+ * @hdev: pointer to habanalabs device structure
+ * @flags: reset flags.
+ *
+ * Block future CS and wait for pending CS to be enqueued
+ * Call ASIC H/W fini
+ * Flush all completions
+ * Re-initialize all internal data structures
+ * Call ASIC H/W init, late_init
+ * Test queues
+ * Enable device
+ *
+ * Returns 0 for success or an error on failure.
+ */
+int hl_device_reset(struct hl_device *hdev, u32 flags)
+{
+	bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false,
+			reset_upon_device_release = false, schedule_hard_reset = false,
+			skip_wq_flush, delay_reset;
+	u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
+	struct hl_ctx *ctx;
+	int i, rc;
+
+	if (!hdev->init_done) {
+		dev_err(hdev->dev, "Can't reset before initialization is done\n");
+		return 0;
+	}
+
+	hard_reset = !!(flags & HL_DRV_RESET_HARD);
+	from_hard_reset_thread = !!(flags & HL_DRV_RESET_FROM_RESET_THR);
+	fw_reset = !!(flags & HL_DRV_RESET_BYPASS_REQ_TO_FW);
+	skip_wq_flush = !!(flags & HL_DRV_RESET_DEV_RELEASE);
+	delay_reset = !!(flags & HL_DRV_RESET_DELAY);
+
+	if (!hard_reset && !hdev->asic_prop.supports_compute_reset) {
+		hard_instead_soft = true;
+		hard_reset = true;
+	}
+
+	if (hdev->reset_upon_device_release && (flags & HL_DRV_RESET_DEV_RELEASE)) {
+		if (hard_reset) {
+			dev_crit(hdev->dev,
+				"Aborting reset because hard-reset is mutually exclusive with reset-on-device-release\n");
+			return -EINVAL;
+		}
+
+		reset_upon_device_release = true;
+
+		goto do_reset;
+	}
+
+	if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) {
+		hard_instead_soft = true;
+		hard_reset = true;
+	}
+
+	if (hard_instead_soft)
+		dev_dbg(hdev->dev, "Doing hard-reset instead of compute reset\n");
+
+do_reset:
+	/* Re-entry of reset thread */
+	if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
+		goto kill_processes;
+
+	/*
+	 * Prevent concurrency in this function - only one reset should be
+	 * done at any given time. Only need to perform this if we didn't
+	 * get from the dedicated hard reset thread
+	 */
+	if (!from_hard_reset_thread) {
+		/* Block future CS/VM/JOB completion operations */
+		spin_lock(&hdev->reset_info.lock);
+		if (hdev->reset_info.in_reset) {
+			/* We only allow scheduling of a hard reset during compute reset */
+			if (hard_reset && hdev->reset_info.in_compute_reset)
+				hdev->reset_info.hard_reset_schedule_flags = flags;
+			spin_unlock(&hdev->reset_info.lock);
+			return 0;
+		}
+
+		/* This still allows the completion of some KDMA ops
+		 * Update this before in_reset because in_compute_reset implies we are in reset
+		 */
+		hdev->reset_info.in_compute_reset = !hard_reset;
+
+		hdev->reset_info.in_reset = 1;
+
+		spin_unlock(&hdev->reset_info.lock);
+
+		if (delay_reset)
+			usleep_range(HL_RESET_DELAY_USEC, HL_RESET_DELAY_USEC << 1);
+
+		handle_reset_trigger(hdev, flags);
+
+		/* This also blocks future CS/VM/JOB completion operations */
+		hdev->disabled = true;
+
+		take_release_locks(hdev);
+
+		if (hard_reset)
+			dev_info(hdev->dev, "Going to reset device\n");
+		else if (reset_upon_device_release)
+			dev_dbg(hdev->dev, "Going to reset device after release by user\n");
+		else
+			dev_dbg(hdev->dev, "Going to reset engines of inference device\n");
+	}
+
+again:
+	if ((hard_reset) && (!from_hard_reset_thread)) {
+		hdev->reset_info.hard_reset_pending = true;
+
+		hdev->process_kill_trial_cnt = 0;
+
+		hdev->device_reset_work.flags = flags;
+
+		/*
+		 * Because the reset function can't run from heartbeat work,
+		 * we need to call the reset function from a dedicated work.
+		 */
+		queue_delayed_work(hdev->device_reset_work.wq,
+			&hdev->device_reset_work.reset_work, 0);
+
+		return 0;
+	}
+
+	cleanup_resources(hdev, hard_reset, fw_reset, skip_wq_flush);
+
+kill_processes:
+	if (hard_reset) {
+		/* Kill processes here after CS rollback. This is because the
+		 * process can't really exit until all its CSs are done, which
+		 * is what we do in cs rollback
+		 */
+		rc = device_kill_open_processes(hdev, 0, false);
+
+		if (rc == -EBUSY) {
+			if (hdev->device_fini_pending) {
+				dev_crit(hdev->dev,
+					"%s Failed to kill all open processes, stopping hard reset\n",
+					dev_name(&(hdev)->pdev->dev));
+				goto out_err;
+			}
+
+			/* signal reset thread to reschedule */
+			return rc;
+		}
+
+		if (rc) {
+			dev_crit(hdev->dev,
+				"%s Failed to kill all open processes, stopping hard reset\n",
+				dev_name(&(hdev)->pdev->dev));
+			goto out_err;
+		}
+
+		/* Flush the Event queue workers to make sure no other thread is
+		 * reading or writing to registers during the reset
+		 */
+		flush_workqueue(hdev->eq_wq);
+	}
+
+	/* Reset the H/W. It will be in idle state after this returns */
+	hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
+
+	if (hard_reset) {
+		hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
+
+		/* Release kernel context */
+		if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
+			hdev->kernel_ctx = NULL;
+
+		hl_vm_fini(hdev);
+		hl_mmu_fini(hdev);
+		hl_eq_reset(hdev, &hdev->event_queue);
+	}
+
+	/* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
+	hl_hw_queue_reset(hdev, hard_reset);
+	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
+		hl_cq_reset(hdev, &hdev->completion_queue[i]);
+
+	/* Make sure the context switch phase will run again */
+	ctx = hl_get_compute_ctx(hdev);
+	if (ctx) {
+		atomic_set(&ctx->thread_ctx_switch_token, 1);
+		ctx->thread_ctx_switch_wait_token = 0;
+		hl_ctx_put(ctx);
+	}
+
+	/* Finished tear-down, starting to re-initialize */
+
+	if (hard_reset) {
+		hdev->device_cpu_disabled = false;
+		hdev->reset_info.hard_reset_pending = false;
+
+		if (hdev->reset_info.reset_trigger_repeated &&
+				(hdev->reset_info.prev_reset_trigger ==
+						HL_DRV_RESET_FW_FATAL_ERR)) {
+			/* if there 2 back to back resets from FW,
+			 * ensure driver puts the driver in a unusable state
+			 */
+			dev_crit(hdev->dev,
+				"%s Consecutive FW fatal errors received, stopping hard reset\n",
+				dev_name(&(hdev)->pdev->dev));
+			rc = -EIO;
+			goto out_err;
+		}
+
+		if (hdev->kernel_ctx) {
+			dev_crit(hdev->dev,
+				"%s kernel ctx was alive during hard reset, something is terribly wrong\n",
+				dev_name(&(hdev)->pdev->dev));
+			rc = -EBUSY;
+			goto out_err;
+		}
+
+		rc = hl_mmu_init(hdev);
+		if (rc) {
+			dev_err(hdev->dev,
+				"Failed to initialize MMU S/W after hard reset\n");
+			goto out_err;
+		}
+
+		/* Allocate the kernel context */
+		hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
+						GFP_KERNEL);
+		if (!hdev->kernel_ctx) {
+			rc = -ENOMEM;
+			hl_mmu_fini(hdev);
+			goto out_err;
+		}
+
+		hdev->is_compute_ctx_active = false;
+
+		rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
+		if (rc) {
+			dev_err(hdev->dev,
+				"failed to init kernel ctx in hard reset\n");
+			kfree(hdev->kernel_ctx);
+			hdev->kernel_ctx = NULL;
+			hl_mmu_fini(hdev);
+			goto out_err;
+		}
+	}
+
+	/* Device is now enabled as part of the initialization requires
+	 * communication with the device firmware to get information that
+	 * is required for the initialization itself
+	 */
+	hdev->disabled = false;
+
+	/* F/W security enabled indication might be updated after hard-reset */
+	if (hard_reset) {
+		rc = hl_fw_read_preboot_status(hdev);
+		if (rc)
+			goto out_err;
+	}
+
+	rc = hdev->asic_funcs->hw_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize the H/W after reset\n");
+		goto out_err;
+	}
+
+	/* If device is not idle fail the reset process */
+	if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
+			HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) {
+		dev_err(hdev->dev, "device is not idle (mask 0x%llx_%llx) after reset\n",
+			idle_mask[1], idle_mask[0]);
+		rc = -EIO;
+		goto out_err;
+	}
+
+	/* Check that the communication with the device is working */
+	rc = hdev->asic_funcs->test_queues(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to detect if device is alive after reset\n");
+		goto out_err;
+	}
+
+	if (hard_reset) {
+		rc = device_late_init(hdev);
+		if (rc) {
+			dev_err(hdev->dev, "Failed late init after hard reset\n");
+			goto out_err;
+		}
+
+		rc = hl_vm_init(hdev);
+		if (rc) {
+			dev_err(hdev->dev, "Failed to init memory module after hard reset\n");
+			goto out_err;
+		}
+
+		if (!hdev->asic_prop.fw_security_enabled)
+			hl_fw_set_max_power(hdev);
+	} else {
+		rc = hdev->asic_funcs->compute_reset_late_init(hdev);
+		if (rc) {
+			if (reset_upon_device_release)
+				dev_err(hdev->dev,
+					"Failed late init in reset after device release\n");
+			else
+				dev_err(hdev->dev, "Failed late init after compute reset\n");
+			goto out_err;
+		}
+	}
+
+	rc = hdev->asic_funcs->scrub_device_mem(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc);
+		return rc;
+	}
+
+	spin_lock(&hdev->reset_info.lock);
+	hdev->reset_info.in_compute_reset = 0;
+
+	/* Schedule hard reset only if requested and if not already in hard reset.
+	 * We keep 'in_reset' enabled, so no other reset can go in during the hard
+	 * reset schedule
+	 */
+	if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags)
+		schedule_hard_reset = true;
+	else
+		hdev->reset_info.in_reset = 0;
+
+	spin_unlock(&hdev->reset_info.lock);
+
+	hdev->reset_info.needs_reset = false;
+
+	if (hard_reset)
+		dev_info(hdev->dev,
+			 "Successfully finished resetting the %s device\n",
+			 dev_name(&(hdev)->pdev->dev));
+	else
+		dev_dbg(hdev->dev,
+			"Successfully finished resetting the %s device\n",
+			dev_name(&(hdev)->pdev->dev));
+
+	if (hard_reset) {
+		hdev->reset_info.hard_reset_cnt++;
+
+		/* After reset is done, we are ready to receive events from
+		 * the F/W. We can't do it before because we will ignore events
+		 * and if those events are fatal, we won't know about it and
+		 * the device will be operational although it shouldn't be
+		 */
+		hdev->asic_funcs->enable_events_from_fw(hdev);
+	} else if (!reset_upon_device_release) {
+		hdev->reset_info.compute_reset_cnt++;
+	}
+
+	if (schedule_hard_reset) {
+		dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n");
+		flags = hdev->reset_info.hard_reset_schedule_flags;
+		hdev->reset_info.hard_reset_schedule_flags = 0;
+		hdev->disabled = true;
+		hard_reset = true;
+		handle_reset_trigger(hdev, flags);
+		goto again;
+	}
+
+	return 0;
+
+out_err:
+	hdev->disabled = true;
+
+	spin_lock(&hdev->reset_info.lock);
+	hdev->reset_info.in_compute_reset = 0;
+
+	if (hard_reset) {
+		dev_err(hdev->dev,
+			"%s Failed to reset! Device is NOT usable\n",
+			dev_name(&(hdev)->pdev->dev));
+		hdev->reset_info.hard_reset_cnt++;
+	} else if (reset_upon_device_release) {
+		spin_unlock(&hdev->reset_info.lock);
+		dev_err(hdev->dev, "Failed to reset device after user release\n");
+		flags |= HL_DRV_RESET_HARD;
+		flags &= ~HL_DRV_RESET_DEV_RELEASE;
+		hard_reset = true;
+		goto again;
+	} else {
+		spin_unlock(&hdev->reset_info.lock);
+		dev_err(hdev->dev, "Failed to do compute reset\n");
+		hdev->reset_info.compute_reset_cnt++;
+		flags |= HL_DRV_RESET_HARD;
+		hard_reset = true;
+		goto again;
+	}
+
+	hdev->reset_info.in_reset = 0;
+
+	spin_unlock(&hdev->reset_info.lock);
+
+	return rc;
+}
+
+static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event_mask)
+{
+	mutex_lock(&notifier_event->lock);
+	notifier_event->events_mask |= event_mask;
+
+	if (notifier_event->eventfd)
+		eventfd_signal(notifier_event->eventfd, 1);
+
+	mutex_unlock(&notifier_event->lock);
+}
+
+/*
+ * hl_notifier_event_send_all - notify all user processes via eventfd
+ *
+ * @hdev: pointer to habanalabs device structure
+ * @event_mask: the occurred event/s
+ * Returns 0 for success or an error on failure.
+ */
+void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask)
+{
+	struct hl_fpriv	*hpriv;
+
+	mutex_lock(&hdev->fpriv_list_lock);
+
+	list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node)
+		hl_notifier_event_send(&hpriv->notifier_event, event_mask);
+
+	mutex_unlock(&hdev->fpriv_list_lock);
+
+	/* control device */
+	mutex_lock(&hdev->fpriv_ctrl_list_lock);
+
+	list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node)
+		hl_notifier_event_send(&hpriv->notifier_event, event_mask);
+
+	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
+}
+
+/*
+ * hl_device_init - main initialization function for habanalabs device
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ * Allocate an id for the device, do early initialization and then call the
+ * ASIC specific initialization functions. Finally, create the cdev and the
+ * Linux device to expose it to the user
+ */
+int hl_device_init(struct hl_device *hdev, struct class *hclass)
+{
+	int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
+	char *name;
+	bool add_cdev_sysfs_on_err = false;
+
+	hdev->cdev_idx = hdev->id / 2;
+
+	name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx);
+	if (!name) {
+		rc = -ENOMEM;
+		goto out_disabled;
+	}
+
+	/* Initialize cdev and device structures */
+	rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,
+				&hdev->cdev, &hdev->dev);
+
+	kfree(name);
+
+	if (rc)
+		goto out_disabled;
+
+	name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx);
+	if (!name) {
+		rc = -ENOMEM;
+		goto free_dev;
+	}
+
+	/* Initialize cdev and device structures for control device */
+	rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
+				name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
+
+	kfree(name);
+
+	if (rc)
+		goto free_dev;
+
+	/* Initialize ASIC function pointers and perform early init */
+	rc = device_early_init(hdev);
+	if (rc)
+		goto free_dev_ctrl;
+
+	user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
+				hdev->asic_prop.user_interrupt_count;
+
+	if (user_interrupt_cnt) {
+		hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt),
+						GFP_KERNEL);
+		if (!hdev->user_interrupt) {
+			rc = -ENOMEM;
+			goto early_fini;
+		}
+	}
+
+	/*
+	 * Start calling ASIC initialization. First S/W then H/W and finally
+	 * late init
+	 */
+	rc = hdev->asic_funcs->sw_init(hdev);
+	if (rc)
+		goto free_usr_intr_mem;
+
+
+	/* initialize completion structure for multi CS wait */
+	hl_multi_cs_completion_init(hdev);
+
+	/*
+	 * Initialize the H/W queues. Must be done before hw_init, because
+	 * there the addresses of the kernel queue are being written to the
+	 * registers of the device
+	 */
+	rc = hl_hw_queues_create(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize kernel queues\n");
+		goto sw_fini;
+	}
+
+	cq_cnt = hdev->asic_prop.completion_queues_count;
+
+	/*
+	 * Initialize the completion queues. Must be done before hw_init,
+	 * because there the addresses of the completion queues are being
+	 * passed as arguments to request_irq
+	 */
+	if (cq_cnt) {
+		hdev->completion_queue = kcalloc(cq_cnt,
+				sizeof(*hdev->completion_queue),
+				GFP_KERNEL);
+
+		if (!hdev->completion_queue) {
+			dev_err(hdev->dev,
+				"failed to allocate completion queues\n");
+			rc = -ENOMEM;
+			goto hw_queues_destroy;
+		}
+	}
+
+	for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
+		rc = hl_cq_init(hdev, &hdev->completion_queue[i],
+				hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
+		if (rc) {
+			dev_err(hdev->dev,
+				"failed to initialize completion queue\n");
+			goto cq_fini;
+		}
+		hdev->completion_queue[i].cq_idx = i;
+	}
+
+	hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs,
+					sizeof(struct hl_cs *), GFP_KERNEL);
+	if (!hdev->shadow_cs_queue) {
+		rc = -ENOMEM;
+		goto cq_fini;
+	}
+
+	/*
+	 * Initialize the event queue. Must be done before hw_init,
+	 * because there the address of the event queue is being
+	 * passed as argument to request_irq
+	 */
+	rc = hl_eq_init(hdev, &hdev->event_queue);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize event queue\n");
+		goto free_shadow_cs_queue;
+	}
+
+	/* MMU S/W must be initialized before kernel context is created */
+	rc = hl_mmu_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
+		goto eq_fini;
+	}
+
+	/* Allocate the kernel context */
+	hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
+	if (!hdev->kernel_ctx) {
+		rc = -ENOMEM;
+		goto mmu_fini;
+	}
+
+	hdev->is_compute_ctx_active = false;
+
+	hdev->asic_funcs->state_dump_init(hdev);
+
+	hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL;
+	hl_debugfs_add_device(hdev);
+
+	/* debugfs nodes are created in hl_ctx_init so it must be called after
+	 * hl_debugfs_add_device.
+	 */
+	rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize kernel context\n");
+		kfree(hdev->kernel_ctx);
+		goto remove_device_from_debugfs;
+	}
+
+	rc = hl_cb_pool_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize CB pool\n");
+		goto release_ctx;
+	}
+
+	rc = hl_dec_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to initialize the decoder module\n");
+		goto cb_pool_fini;
+	}
+
+	/*
+	 * From this point, override rc (=0) in case of an error to allow
+	 * debugging (by adding char devices and create sysfs nodes as part of
+	 * the error flow).
+	 */
+	add_cdev_sysfs_on_err = true;
+
+	/* Device is now enabled as part of the initialization requires
+	 * communication with the device firmware to get information that
+	 * is required for the initialization itself
+	 */
+	hdev->disabled = false;
+
+	rc = hdev->asic_funcs->hw_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize the H/W\n");
+		rc = 0;
+		goto out_disabled;
+	}
+
+	/* Check that the communication with the device is working */
+	rc = hdev->asic_funcs->test_queues(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to detect if device is alive\n");
+		rc = 0;
+		goto out_disabled;
+	}
+
+	rc = device_late_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed late initialization\n");
+		rc = 0;
+		goto out_disabled;
+	}
+
+	dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
+		hdev->asic_name,
+		hdev->asic_prop.dram_size / SZ_1G);
+
+	rc = hl_vm_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to initialize memory module\n");
+		rc = 0;
+		goto out_disabled;
+	}
+
+	/*
+	 * Expose devices and sysfs nodes to user.
+	 * From here there is no need to add char devices and create sysfs nodes
+	 * in case of an error.
+	 */
+	add_cdev_sysfs_on_err = false;
+	rc = device_cdev_sysfs_add(hdev);
+	if (rc) {
+		dev_err(hdev->dev,
+			"Failed to add char devices and sysfs nodes\n");
+		rc = 0;
+		goto out_disabled;
+	}
+
+	/* Need to call this again because the max power might change,
+	 * depending on card type for certain ASICs
+	 */
+	if (hdev->asic_prop.set_max_power_on_device_init &&
+			!hdev->asic_prop.fw_security_enabled)
+		hl_fw_set_max_power(hdev);
+
+	/*
+	 * hl_hwmon_init() must be called after device_late_init(), because only
+	 * there we get the information from the device about which
+	 * hwmon-related sensors the device supports.
+	 * Furthermore, it must be done after adding the device to the system.
+	 */
+	rc = hl_hwmon_init(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to initialize hwmon\n");
+		rc = 0;
+		goto out_disabled;
+	}
+
+	dev_notice(hdev->dev,
+		"Successfully added device %s to habanalabs driver\n",
+		dev_name(&(hdev)->pdev->dev));
+
+	hdev->init_done = true;
+
+	/* After initialization is done, we are ready to receive events from
+	 * the F/W. We can't do it before because we will ignore events and if
+	 * those events are fatal, we won't know about it and the device will
+	 * be operational although it shouldn't be
+	 */
+	hdev->asic_funcs->enable_events_from_fw(hdev);
+
+	return 0;
+
+cb_pool_fini:
+	hl_cb_pool_fini(hdev);
+release_ctx:
+	if (hl_ctx_put(hdev->kernel_ctx) != 1)
+		dev_err(hdev->dev,
+			"kernel ctx is still alive on initialization failure\n");
+remove_device_from_debugfs:
+	hl_debugfs_remove_device(hdev);
+mmu_fini:
+	hl_mmu_fini(hdev);
+eq_fini:
+	hl_eq_fini(hdev, &hdev->event_queue);
+free_shadow_cs_queue:
+	kfree(hdev->shadow_cs_queue);
+cq_fini:
+	for (i = 0 ; i < cq_ready_cnt ; i++)
+		hl_cq_fini(hdev, &hdev->completion_queue[i]);
+	kfree(hdev->completion_queue);
+hw_queues_destroy:
+	hl_hw_queues_destroy(hdev);
+sw_fini:
+	hdev->asic_funcs->sw_fini(hdev);
+free_usr_intr_mem:
+	kfree(hdev->user_interrupt);
+early_fini:
+	device_early_fini(hdev);
+free_dev_ctrl:
+	put_device(hdev->dev_ctrl);
+free_dev:
+	put_device(hdev->dev);
+out_disabled:
+	hdev->disabled = true;
+	if (add_cdev_sysfs_on_err)
+		device_cdev_sysfs_add(hdev);
+	if (hdev->pdev)
+		dev_err(&hdev->pdev->dev,
+			"Failed to initialize hl%d. Device %s is NOT usable !\n",
+			hdev->cdev_idx, dev_name(&(hdev)->pdev->dev));
+	else
+		pr_err("Failed to initialize hl%d. Device %s is NOT usable !\n",
+			hdev->cdev_idx, dev_name(&(hdev)->pdev->dev));
+
+	return rc;
+}
+
+/*
+ * hl_device_fini - main tear-down function for habanalabs device
+ *
+ * @hdev: pointer to habanalabs device structure
+ *
+ * Destroy the device, call ASIC fini functions and release the id
+ */
+void hl_device_fini(struct hl_device *hdev)
+{
+	bool device_in_reset;
+	ktime_t timeout;
+	u64 reset_sec;
+	int i, rc;
+
+	dev_info(hdev->dev, "Removing device\n");
+
+	hdev->device_fini_pending = 1;
+	flush_delayed_work(&hdev->device_reset_work.reset_work);
+
+	if (hdev->pldm)
+		reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT;
+	else
+		reset_sec = HL_HARD_RESET_MAX_TIMEOUT;
+
+	/*
+	 * This function is competing with the reset function, so try to
+	 * take the reset atomic and if we are already in middle of reset,
+	 * wait until reset function is finished. Reset function is designed
+	 * to always finish. However, in Gaudi, because of all the network
+	 * ports, the hard reset could take between 10-30 seconds
+	 */
+
+	timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000);
+
+	spin_lock(&hdev->reset_info.lock);
+	device_in_reset = !!hdev->reset_info.in_reset;
+	if (!device_in_reset)
+		hdev->reset_info.in_reset = 1;
+	spin_unlock(&hdev->reset_info.lock);
+
+	while (device_in_reset) {
+		usleep_range(50, 200);
+
+		spin_lock(&hdev->reset_info.lock);
+		device_in_reset = !!hdev->reset_info.in_reset;
+		if (!device_in_reset)
+			hdev->reset_info.in_reset = 1;
+		spin_unlock(&hdev->reset_info.lock);
+
+		if (ktime_compare(ktime_get(), timeout) > 0) {
+			dev_crit(hdev->dev,
+				"%s Failed to remove device because reset function did not finish\n",
+				dev_name(&(hdev)->pdev->dev));
+			return;
+		}
+	}
+
+	/* Disable PCI access from device F/W so it won't send us additional
+	 * interrupts. We disable MSI/MSI-X at the halt_engines function and we
+	 * can't have the F/W sending us interrupts after that. We need to
+	 * disable the access here because if the device is marked disable, the
+	 * message won't be send. Also, in case of heartbeat, the device CPU is
+	 * marked as disable so this message won't be sent
+	 */
+	hl_fw_send_pci_access_msg(hdev,	CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
+
+	/* Mark device as disabled */
+	hdev->disabled = true;
+
+	take_release_locks(hdev);
+
+	hdev->reset_info.hard_reset_pending = true;
+
+	hl_hwmon_fini(hdev);
+
+	cleanup_resources(hdev, true, false, false);
+
+	/* Kill processes here after CS rollback. This is because the process
+	 * can't really exit until all its CSs are done, which is what we
+	 * do in cs rollback
+	 */
+	dev_info(hdev->dev,
+		"Waiting for all processes to exit (timeout of %u seconds)",
+		HL_PENDING_RESET_LONG_SEC);
+
+	rc = device_kill_open_processes(hdev, HL_PENDING_RESET_LONG_SEC, false);
+	if (rc) {
+		dev_crit(hdev->dev, "Failed to kill all open processes\n");
+		device_disable_open_processes(hdev, false);
+	}
+
+	rc = device_kill_open_processes(hdev, 0, true);
+	if (rc) {
+		dev_crit(hdev->dev, "Failed to kill all control device open processes\n");
+		device_disable_open_processes(hdev, true);
+	}
+
+	hl_cb_pool_fini(hdev);
+
+	/* Reset the H/W. It will be in idle state after this returns */
+	hdev->asic_funcs->hw_fini(hdev, true, false);
+
+	hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
+
+	/* Release kernel context */
+	if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
+		dev_err(hdev->dev, "kernel ctx is still alive\n");
+
+	hl_debugfs_remove_device(hdev);
+
+	hl_dec_fini(hdev);
+
+	hl_vm_fini(hdev);
+
+	hl_mmu_fini(hdev);
+
+	hl_eq_fini(hdev, &hdev->event_queue);
+
+	kfree(hdev->shadow_cs_queue);
+
+	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
+		hl_cq_fini(hdev, &hdev->completion_queue[i]);
+	kfree(hdev->completion_queue);
+	kfree(hdev->user_interrupt);
+
+	hl_hw_queues_destroy(hdev);
+
+	/* Call ASIC S/W finalize function */
+	hdev->asic_funcs->sw_fini(hdev);
+
+	device_early_fini(hdev);
+
+	/* Hide devices and sysfs nodes from user */
+	device_cdev_sysfs_del(hdev);
+
+	pr_info("removed device successfully\n");
+}
+
+/*
+ * MMIO register access helper functions.
+ */
+
+/*
+ * hl_rreg - Read an MMIO register
+ *
+ * @hdev: pointer to habanalabs device structure
+ * @reg: MMIO register offset (in bytes)
+ *
+ * Returns the value of the MMIO register we are asked to read
+ *
+ */
+inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
+{
+	return readl(hdev->rmmio + reg);
+}
+
+/*
+ * hl_wreg - Write to an MMIO register
+ *
+ * @hdev: pointer to habanalabs device structure
+ * @reg: MMIO register offset (in bytes)
+ * @val: 32-bit value
+ *
+ * Writes the 32-bit value into the MMIO register
+ *
+ */
+inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
+{
+	writel(val, hdev->rmmio + reg);
+}