diff options
Diffstat (limited to 'src/VBox/Devices/EFI/Firmware/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c')
| -rw-r--r-- | src/VBox/Devices/EFI/Firmware/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c | 1126 |
1 files changed, 1126 insertions, 0 deletions
diff --git a/src/VBox/Devices/EFI/Firmware/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c b/src/VBox/Devices/EFI/Firmware/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c new file mode 100644 index 00000000..65b97ee6 --- /dev/null +++ b/src/VBox/Devices/EFI/Firmware/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c @@ -0,0 +1,1126 @@ +/** @file + NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows + NVM Express specification. + + Copyright (c) 2013 - 2019, Intel Corporation. All rights reserved.<BR> + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#include "NvmExpress.h" + +#define NVME_SHUTDOWN_PROCESS_TIMEOUT 45 + +// +// The number of NVME controllers managed by this driver, used by +// NvmeRegisterShutdownNotification() and NvmeUnregisterShutdownNotification(). +// +UINTN mNvmeControllerNumber = 0; + +/** + Read Nvm Express controller capability register. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param Cap The buffer used to store capability register content. + + @return EFI_SUCCESS Successfully read the controller capability register content. + @return EFI_DEVICE_ERROR Fail to read the controller capability register. + +**/ +EFI_STATUS +ReadNvmeControllerCapabilities ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN NVME_CAP *Cap + ) +{ + EFI_PCI_IO_PROTOCOL *PciIo; + EFI_STATUS Status; + UINT64 Data; + + PciIo = Private->PciIo; + Status = PciIo->Mem.Read ( + PciIo, + EfiPciIoWidthUint32, + NVME_BAR, + NVME_CAP_OFFSET, + 2, + &Data + ); + + if (EFI_ERROR(Status)) { + return Status; + } + + WriteUnaligned64 ((UINT64*)Cap, Data); + return EFI_SUCCESS; +} + +/** + Read Nvm Express controller configuration register. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param Cc The buffer used to store configuration register content. + + @return EFI_SUCCESS Successfully read the controller configuration register content. + @return EFI_DEVICE_ERROR Fail to read the controller configuration register. + +**/ +EFI_STATUS +ReadNvmeControllerConfiguration ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN NVME_CC *Cc + ) +{ + EFI_PCI_IO_PROTOCOL *PciIo; + EFI_STATUS Status; + UINT32 Data; + + PciIo = Private->PciIo; + Status = PciIo->Mem.Read ( + PciIo, + EfiPciIoWidthUint32, + NVME_BAR, + NVME_CC_OFFSET, + 1, + &Data + ); + + if (EFI_ERROR(Status)) { + return Status; + } + + WriteUnaligned32 ((UINT32*)Cc, Data); + return EFI_SUCCESS; +} + +/** + Write Nvm Express controller configuration register. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param Cc The buffer used to store the content to be written into configuration register. + + @return EFI_SUCCESS Successfully write data into the controller configuration register. + @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register. + +**/ +EFI_STATUS +WriteNvmeControllerConfiguration ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN NVME_CC *Cc + ) +{ + EFI_PCI_IO_PROTOCOL *PciIo; + EFI_STATUS Status; + UINT32 Data; + + PciIo = Private->PciIo; + Data = ReadUnaligned32 ((UINT32*)Cc); + Status = PciIo->Mem.Write ( + PciIo, + EfiPciIoWidthUint32, + NVME_BAR, + NVME_CC_OFFSET, + 1, + &Data + ); + + if (EFI_ERROR(Status)) { + return Status; + } + + DEBUG ((EFI_D_INFO, "Cc.En: %d\n", Cc->En)); + DEBUG ((EFI_D_INFO, "Cc.Css: %d\n", Cc->Css)); + DEBUG ((EFI_D_INFO, "Cc.Mps: %d\n", Cc->Mps)); + DEBUG ((EFI_D_INFO, "Cc.Ams: %d\n", Cc->Ams)); + DEBUG ((EFI_D_INFO, "Cc.Shn: %d\n", Cc->Shn)); + DEBUG ((EFI_D_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes)); + DEBUG ((EFI_D_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes)); + + return EFI_SUCCESS; +} + +/** + Read Nvm Express controller status register. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param Csts The buffer used to store status register content. + + @return EFI_SUCCESS Successfully read the controller status register content. + @return EFI_DEVICE_ERROR Fail to read the controller status register. + +**/ +EFI_STATUS +ReadNvmeControllerStatus ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN NVME_CSTS *Csts + ) +{ + EFI_PCI_IO_PROTOCOL *PciIo; + EFI_STATUS Status; + UINT32 Data; + + PciIo = Private->PciIo; + Status = PciIo->Mem.Read ( + PciIo, + EfiPciIoWidthUint32, + NVME_BAR, + NVME_CSTS_OFFSET, + 1, + &Data + ); + + if (EFI_ERROR(Status)) { + return Status; + } + + WriteUnaligned32 ((UINT32*)Csts, Data); + return EFI_SUCCESS; +} + + + +/** + Write Nvm Express admin queue attributes register. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param Aqa The buffer used to store the content to be written into admin queue attributes register. + + @return EFI_SUCCESS Successfully write data into the admin queue attributes register. + @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register. + +**/ +EFI_STATUS +WriteNvmeAdminQueueAttributes ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN NVME_AQA *Aqa + ) +{ + EFI_PCI_IO_PROTOCOL *PciIo; + EFI_STATUS Status; + UINT32 Data; + + PciIo = Private->PciIo; + Data = ReadUnaligned32 ((UINT32*)Aqa); + Status = PciIo->Mem.Write ( + PciIo, + EfiPciIoWidthUint32, + NVME_BAR, + NVME_AQA_OFFSET, + 1, + &Data + ); + + if (EFI_ERROR(Status)) { + return Status; + } + + DEBUG ((EFI_D_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs)); + DEBUG ((EFI_D_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs)); + + return EFI_SUCCESS; +} + + +/** + Write Nvm Express admin submission queue base address register. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param Asq The buffer used to store the content to be written into admin submission queue base address register. + + @return EFI_SUCCESS Successfully write data into the admin submission queue base address register. + @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register. + +**/ +EFI_STATUS +WriteNvmeAdminSubmissionQueueBaseAddress ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN NVME_ASQ *Asq + ) +{ + EFI_PCI_IO_PROTOCOL *PciIo; + EFI_STATUS Status; + UINT64 Data; + + PciIo = Private->PciIo; + Data = ReadUnaligned64 ((UINT64*)Asq); + + Status = PciIo->Mem.Write ( + PciIo, + EfiPciIoWidthUint32, + NVME_BAR, + NVME_ASQ_OFFSET, + 2, + &Data + ); + + if (EFI_ERROR(Status)) { + return Status; + } + + DEBUG ((EFI_D_INFO, "Asq: %lx\n", *Asq)); + + return EFI_SUCCESS; +} + + + +/** + Write Nvm Express admin completion queue base address register. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param Acq The buffer used to store the content to be written into admin completion queue base address register. + + @return EFI_SUCCESS Successfully write data into the admin completion queue base address register. + @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register. + +**/ +EFI_STATUS +WriteNvmeAdminCompletionQueueBaseAddress ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN NVME_ACQ *Acq + ) +{ + EFI_PCI_IO_PROTOCOL *PciIo; + EFI_STATUS Status; + UINT64 Data; + + PciIo = Private->PciIo; + Data = ReadUnaligned64 ((UINT64*)Acq); + + Status = PciIo->Mem.Write ( + PciIo, + EfiPciIoWidthUint32, + NVME_BAR, + NVME_ACQ_OFFSET, + 2, + &Data + ); + + if (EFI_ERROR(Status)) { + return Status; + } + + DEBUG ((EFI_D_INFO, "Acq: %lxh\n", *Acq)); + + return EFI_SUCCESS; +} + +/** + Disable the Nvm Express controller. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + + @return EFI_SUCCESS Successfully disable the controller. + @return EFI_DEVICE_ERROR Fail to disable the controller. + +**/ +EFI_STATUS +NvmeDisableController ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private + ) +{ + NVME_CC Cc; + NVME_CSTS Csts; + EFI_STATUS Status; + UINT32 Index; + UINT8 Timeout; + + // + // Read Controller Configuration Register. + // + Status = ReadNvmeControllerConfiguration (Private, &Cc); + if (EFI_ERROR(Status)) { + return Status; + } + + Cc.En = 0; + + // + // Disable the controller. + // + Status = WriteNvmeControllerConfiguration (Private, &Cc); + + if (EFI_ERROR(Status)) { + return Status; + } + + // + // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after + // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To. + // + if (Private->Cap.To == 0) { + Timeout = 1; + } else { + Timeout = Private->Cap.To; + } + + for(Index = (Timeout * 500); Index != 0; --Index) { + gBS->Stall(1000); + + // + // Check if the controller is initialized + // + Status = ReadNvmeControllerStatus (Private, &Csts); + + if (EFI_ERROR(Status)) { + return Status; + } + + if (Csts.Rdy == 0) { + break; + } + } + + if (Index == 0) { + Status = EFI_DEVICE_ERROR; + REPORT_STATUS_CODE ( + (EFI_ERROR_CODE | EFI_ERROR_MAJOR), + (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR) + ); + } + + DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status)); + return Status; +} + +/** + Enable the Nvm Express controller. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + + @return EFI_SUCCESS Successfully enable the controller. + @return EFI_DEVICE_ERROR Fail to enable the controller. + @return EFI_TIMEOUT Fail to enable the controller in given time slot. + +**/ +EFI_STATUS +NvmeEnableController ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private + ) +{ + NVME_CC Cc; + NVME_CSTS Csts; + EFI_STATUS Status; + UINT32 Index; + UINT8 Timeout; + + // + // Enable the controller. + // CC.AMS, CC.MPS and CC.CSS are all set to 0. + // + ZeroMem (&Cc, sizeof (NVME_CC)); + Cc.En = 1; + Cc.Iosqes = 6; + Cc.Iocqes = 4; + + Status = WriteNvmeControllerConfiguration (Private, &Cc); + if (EFI_ERROR(Status)) { + return Status; + } + + // + // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after + // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To. + // + if (Private->Cap.To == 0) { + Timeout = 1; + } else { + Timeout = Private->Cap.To; + } + + for(Index = (Timeout * 500); Index != 0; --Index) { + gBS->Stall(1000); + + // + // Check if the controller is initialized + // + Status = ReadNvmeControllerStatus (Private, &Csts); + + if (EFI_ERROR(Status)) { + return Status; + } + + if (Csts.Rdy) { + break; + } + } + + if (Index == 0) { + Status = EFI_TIMEOUT; + REPORT_STATUS_CODE ( + (EFI_ERROR_CODE | EFI_ERROR_MAJOR), + (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR) + ); + } + + DEBUG ((EFI_D_INFO, "NVMe controller is enabled with status [%r].\n", Status)); + return Status; +} + +/** + Get identify controller data. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param Buffer The buffer used to store the identify controller data. + + @return EFI_SUCCESS Successfully get the identify controller data. + @return EFI_DEVICE_ERROR Fail to get the identify controller data. + +**/ +EFI_STATUS +NvmeIdentifyController ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN VOID *Buffer + ) +{ + EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; + EFI_NVM_EXPRESS_COMMAND Command; + EFI_NVM_EXPRESS_COMPLETION Completion; + EFI_STATUS Status; + + ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); + ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); + ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); + + Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD; + // + // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h. + // For the Identify command, the Namespace Identifier is only used for the Namespace data structure. + // + Command.Nsid = 0; + + CommandPacket.NvmeCmd = &Command; + CommandPacket.NvmeCompletion = &Completion; + CommandPacket.TransferBuffer = Buffer; + CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA); + CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; + CommandPacket.QueueType = NVME_ADMIN_QUEUE; + // + // Set bit 0 (Cns bit) to 1 to identify a controller + // + Command.Cdw10 = 1; + Command.Flags = CDW10_VALID; + + Status = Private->Passthru.PassThru ( + &Private->Passthru, + NVME_CONTROLLER_ID, + &CommandPacket, + NULL + ); + + return Status; +} + +/** + Get specified identify namespace data. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + @param NamespaceId The specified namespace identifier. + @param Buffer The buffer used to store the identify namespace data. + + @return EFI_SUCCESS Successfully get the identify namespace data. + @return EFI_DEVICE_ERROR Fail to get the identify namespace data. + +**/ +EFI_STATUS +NvmeIdentifyNamespace ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private, + IN UINT32 NamespaceId, + IN VOID *Buffer + ) +{ + EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; + EFI_NVM_EXPRESS_COMMAND Command; + EFI_NVM_EXPRESS_COMPLETION Completion; + EFI_STATUS Status; + + ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); + ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); + ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); + + CommandPacket.NvmeCmd = &Command; + CommandPacket.NvmeCompletion = &Completion; + + Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD; + Command.Nsid = NamespaceId; + CommandPacket.TransferBuffer = Buffer; + CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA); + CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; + CommandPacket.QueueType = NVME_ADMIN_QUEUE; + // + // Set bit 0 (Cns bit) to 1 to identify a namespace + // + CommandPacket.NvmeCmd->Cdw10 = 0; + CommandPacket.NvmeCmd->Flags = CDW10_VALID; + + Status = Private->Passthru.PassThru ( + &Private->Passthru, + NamespaceId, + &CommandPacket, + NULL + ); + + return Status; +} + +/** + Create io completion queue. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + + @return EFI_SUCCESS Successfully create io completion queue. + @return EFI_DEVICE_ERROR Fail to create io completion queue. + +**/ +EFI_STATUS +NvmeCreateIoCompletionQueue ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private + ) +{ + EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; + EFI_NVM_EXPRESS_COMMAND Command; + EFI_NVM_EXPRESS_COMPLETION Completion; + EFI_STATUS Status; + NVME_ADMIN_CRIOCQ CrIoCq; + UINT32 Index; + UINT16 QueueSize; + + Status = EFI_SUCCESS; + Private->CreateIoQueue = TRUE; + + for (Index = 1; Index < NVME_MAX_QUEUES; Index++) { + ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); + ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); + ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); + ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ)); + + CommandPacket.NvmeCmd = &Command; + CommandPacket.NvmeCompletion = &Completion; + + Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD; + CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index]; + CommandPacket.TransferLength = EFI_PAGE_SIZE; + CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; + CommandPacket.QueueType = NVME_ADMIN_QUEUE; + + if (Index == 1) { + QueueSize = NVME_CCQ_SIZE; + } else { + if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { + QueueSize = NVME_ASYNC_CCQ_SIZE; + } else { + QueueSize = Private->Cap.Mqes; + } + } + + CrIoCq.Qid = Index; + CrIoCq.Qsize = QueueSize; + CrIoCq.Pc = 1; + CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ)); + CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID; + + Status = Private->Passthru.PassThru ( + &Private->Passthru, + 0, + &CommandPacket, + NULL + ); + if (EFI_ERROR (Status)) { + break; + } + } + + Private->CreateIoQueue = FALSE; + + return Status; +} + +/** + Create io submission queue. + + @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + + @return EFI_SUCCESS Successfully create io submission queue. + @return EFI_DEVICE_ERROR Fail to create io submission queue. + +**/ +EFI_STATUS +NvmeCreateIoSubmissionQueue ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private + ) +{ + EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; + EFI_NVM_EXPRESS_COMMAND Command; + EFI_NVM_EXPRESS_COMPLETION Completion; + EFI_STATUS Status; + NVME_ADMIN_CRIOSQ CrIoSq; + UINT32 Index; + UINT16 QueueSize; + + Status = EFI_SUCCESS; + Private->CreateIoQueue = TRUE; + + for (Index = 1; Index < NVME_MAX_QUEUES; Index++) { + ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); + ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); + ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); + ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ)); + + CommandPacket.NvmeCmd = &Command; + CommandPacket.NvmeCompletion = &Completion; + + Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD; + CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index]; + CommandPacket.TransferLength = EFI_PAGE_SIZE; + CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; + CommandPacket.QueueType = NVME_ADMIN_QUEUE; + + if (Index == 1) { + QueueSize = NVME_CSQ_SIZE; + } else { + if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) { + QueueSize = NVME_ASYNC_CSQ_SIZE; + } else { + QueueSize = Private->Cap.Mqes; + } + } + + CrIoSq.Qid = Index; + CrIoSq.Qsize = QueueSize; + CrIoSq.Pc = 1; + CrIoSq.Cqid = Index; + CrIoSq.Qprio = 0; + CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ)); + CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID; + + Status = Private->Passthru.PassThru ( + &Private->Passthru, + 0, + &CommandPacket, + NULL + ); + if (EFI_ERROR (Status)) { + break; + } + } + + Private->CreateIoQueue = FALSE; + + return Status; +} + +/** + Initialize the Nvm Express controller. + + @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. + + @retval EFI_SUCCESS The NVM Express Controller is initialized successfully. + @retval Others A device error occurred while initializing the controller. + +**/ +EFI_STATUS +NvmeControllerInit ( + IN NVME_CONTROLLER_PRIVATE_DATA *Private + ) +{ + EFI_STATUS Status; + EFI_PCI_IO_PROTOCOL *PciIo; + UINT64 Supports; + NVME_AQA Aqa; + NVME_ASQ Asq; + NVME_ACQ Acq; + UINT8 Sn[21]; + UINT8 Mn[41]; + // + // Save original PCI attributes and enable this controller. + // + PciIo = Private->PciIo; + Status = PciIo->Attributes ( + PciIo, + EfiPciIoAttributeOperationGet, + 0, + &Private->PciAttributes + ); + + if (EFI_ERROR (Status)) { + return Status; + } + + Status = PciIo->Attributes ( + PciIo, + EfiPciIoAttributeOperationSupported, + 0, + &Supports + ); + + if (!EFI_ERROR (Status)) { + Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE; + Status = PciIo->Attributes ( + PciIo, + EfiPciIoAttributeOperationEnable, + Supports, + NULL + ); + } + + if (EFI_ERROR (Status)) { + DEBUG ((EFI_D_INFO, "NvmeControllerInit: failed to enable controller\n")); + return Status; + } + + // + // Enable 64-bit DMA support in the PCI layer. + // + Status = PciIo->Attributes ( + PciIo, + EfiPciIoAttributeOperationEnable, + EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE, + NULL + ); + if (EFI_ERROR (Status)) { + DEBUG ((EFI_D_WARN, "NvmeControllerInit: failed to enable 64-bit DMA (%r)\n", Status)); + } + + // + // Read the Controller Capabilities register and verify that the NVM command set is supported + // + Status = ReadNvmeControllerCapabilities (Private, &Private->Cap); + if (EFI_ERROR (Status)) { + return Status; + } + + if (Private->Cap.Css != 0x01) { + DEBUG ((EFI_D_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n")); + return EFI_UNSUPPORTED; + } + + // + // Currently the driver only supports 4k page size. + // + ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT); + + Private->Cid[0] = 0; + Private->Cid[1] = 0; + Private->Cid[2] = 0; + Private->Pt[0] = 0; + Private->Pt[1] = 0; + Private->Pt[2] = 0; + Private->SqTdbl[0].Sqt = 0; + Private->SqTdbl[1].Sqt = 0; + Private->SqTdbl[2].Sqt = 0; + Private->CqHdbl[0].Cqh = 0; + Private->CqHdbl[1].Cqh = 0; + Private->CqHdbl[2].Cqh = 0; + Private->AsyncSqHead = 0; + + Status = NvmeDisableController (Private); + + if (EFI_ERROR(Status)) { + return Status; + } + + // + // set number of entries admin submission & completion queues. + // + Aqa.Asqs = NVME_ASQ_SIZE; + Aqa.Rsvd1 = 0; + Aqa.Acqs = NVME_ACQ_SIZE; + Aqa.Rsvd2 = 0; + + // + // Address of admin submission queue. + // + Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF; + + // + // Address of admin completion queue. + // + Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF; + + // + // Address of I/O submission & completion queue. + // + ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6)); + Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); + Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); + Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE); + Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE); + Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE); + Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); + Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); + Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); + Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); + + DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer)); + DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs)); + DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs)); + DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0])); + DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0])); + DEBUG ((EFI_D_INFO, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1])); + DEBUG ((EFI_D_INFO, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1])); + DEBUG ((EFI_D_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2])); + DEBUG ((EFI_D_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2])); + + // + // Program admin queue attributes. + // + Status = WriteNvmeAdminQueueAttributes (Private, &Aqa); + + if (EFI_ERROR(Status)) { + return Status; + } + + // + // Program admin submission queue address. + // + Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq); + + if (EFI_ERROR(Status)) { + return Status; + } + + // + // Program admin completion queue address. + // + Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq); + + if (EFI_ERROR(Status)) { + return Status; + } + + Status = NvmeEnableController (Private); + if (EFI_ERROR(Status)) { + return Status; + } + + // + // Allocate buffer for Identify Controller data + // + if (Private->ControllerData == NULL) { + Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA)); + + if (Private->ControllerData == NULL) { + return EFI_OUT_OF_RESOURCES; + } + } + + // + // Get current Identify Controller Data + // + Status = NvmeIdentifyController (Private, Private->ControllerData); + + if (EFI_ERROR(Status)) { + FreePool(Private->ControllerData); + Private->ControllerData = NULL; + return EFI_NOT_FOUND; + } + + // + // Dump NvmExpress Identify Controller Data + // + CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn)); + Sn[20] = 0; + CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn)); + Mn[40] = 0; + DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n")); + DEBUG ((EFI_D_INFO, " PCI VID : 0x%x\n", Private->ControllerData->Vid)); + DEBUG ((EFI_D_INFO, " PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid)); + DEBUG ((EFI_D_INFO, " SN : %a\n", Sn)); + DEBUG ((EFI_D_INFO, " MN : %a\n", Mn)); + DEBUG ((EFI_D_INFO, " FR : 0x%x\n", *((UINT64*)Private->ControllerData->Fr))); + DEBUG ((EFI_D_INFO, " RAB : 0x%x\n", Private->ControllerData->Rab)); + DEBUG ((EFI_D_INFO, " IEEE : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oui)); + DEBUG ((EFI_D_INFO, " AERL : 0x%x\n", Private->ControllerData->Aerl)); + DEBUG ((EFI_D_INFO, " SQES : 0x%x\n", Private->ControllerData->Sqes)); + DEBUG ((EFI_D_INFO, " CQES : 0x%x\n", Private->ControllerData->Cqes)); + DEBUG ((EFI_D_INFO, " NN : 0x%x\n", Private->ControllerData->Nn)); + + // + // Create two I/O completion queues. + // One for blocking I/O, one for non-blocking I/O. + // + Status = NvmeCreateIoCompletionQueue (Private); + if (EFI_ERROR(Status)) { + return Status; + } + + // + // Create two I/O Submission queues. + // One for blocking I/O, one for non-blocking I/O. + // + Status = NvmeCreateIoSubmissionQueue (Private); + + return Status; +} + +/** + This routine is called to properly shutdown the Nvm Express controller per NVMe spec. + + @param[in] ResetType The type of reset to perform. + @param[in] ResetStatus The status code for the reset. + @param[in] DataSize The size, in bytes, of ResetData. + @param[in] ResetData For a ResetType of EfiResetCold, EfiResetWarm, or + EfiResetShutdown the data buffer starts with a Null-terminated + string, optionally followed by additional binary data. + The string is a description that the caller may use to further + indicate the reason for the system reset. + For a ResetType of EfiResetPlatformSpecific the data buffer + also starts with a Null-terminated string that is followed + by an EFI_GUID that describes the specific type of reset to perform. +**/ +VOID +EFIAPI +NvmeShutdownAllControllers ( + IN EFI_RESET_TYPE ResetType, + IN EFI_STATUS ResetStatus, + IN UINTN DataSize, + IN VOID *ResetData OPTIONAL + ) +{ + EFI_STATUS Status; + EFI_HANDLE *Handles; + UINTN HandleCount; + UINTN HandleIndex; + EFI_OPEN_PROTOCOL_INFORMATION_ENTRY *OpenInfos; + UINTN OpenInfoCount; + UINTN OpenInfoIndex; + EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *NvmePassThru; + NVME_CC Cc; + NVME_CSTS Csts; + UINTN Index; + NVME_CONTROLLER_PRIVATE_DATA *Private; + + Status = gBS->LocateHandleBuffer ( + ByProtocol, + &gEfiPciIoProtocolGuid, + NULL, + &HandleCount, + &Handles + ); + if (EFI_ERROR (Status)) { + HandleCount = 0; + } + + for (HandleIndex = 0; HandleIndex < HandleCount; HandleIndex++) { + Status = gBS->OpenProtocolInformation ( + Handles[HandleIndex], + &gEfiPciIoProtocolGuid, + &OpenInfos, + &OpenInfoCount + ); + if (EFI_ERROR (Status)) { + continue; + } + + for (OpenInfoIndex = 0; OpenInfoIndex < OpenInfoCount; OpenInfoIndex++) { + // + // Find all the NVME controller managed by this driver. + // gImageHandle equals to DriverBinding handle for this driver. + // + if (((OpenInfos[OpenInfoIndex].Attributes & EFI_OPEN_PROTOCOL_BY_DRIVER) != 0) && + (OpenInfos[OpenInfoIndex].AgentHandle == gImageHandle)) { + Status = gBS->OpenProtocol ( + OpenInfos[OpenInfoIndex].ControllerHandle, + &gEfiNvmExpressPassThruProtocolGuid, + (VOID **) &NvmePassThru, + NULL, + NULL, + EFI_OPEN_PROTOCOL_GET_PROTOCOL + ); + if (EFI_ERROR (Status)) { + continue; + } + Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (NvmePassThru); + + // + // Read Controller Configuration Register. + // + Status = ReadNvmeControllerConfiguration (Private, &Cc); + if (EFI_ERROR(Status)) { + continue; + } + // + // The host should set the Shutdown Notification (CC.SHN) field to 01b + // to indicate a normal shutdown operation. + // + Cc.Shn = NVME_CC_SHN_NORMAL_SHUTDOWN; + Status = WriteNvmeControllerConfiguration (Private, &Cc); + if (EFI_ERROR(Status)) { + continue; + } + + // + // The controller indicates when shutdown processing is completed by updating the + // Shutdown Status (CSTS.SHST) field to 10b. + // Wait up to 45 seconds (break down to 4500 x 10ms) for the shutdown to complete. + // + for (Index = 0; Index < NVME_SHUTDOWN_PROCESS_TIMEOUT * 100; Index++) { + Status = ReadNvmeControllerStatus (Private, &Csts); + if (!EFI_ERROR(Status) && (Csts.Shst == NVME_CSTS_SHST_SHUTDOWN_COMPLETED)) { + DEBUG((DEBUG_INFO, "NvmeShutdownController: shutdown processing is completed after %dms.\n", Index * 10)); + break; + } + // + // Stall for 10ms + // + gBS->Stall (10 * 1000); + } + + if (Index == NVME_SHUTDOWN_PROCESS_TIMEOUT * 100) { + DEBUG((DEBUG_ERROR, "NvmeShutdownController: shutdown processing is timed out\n")); + } + } + } + } +} + +/** + Register the shutdown notification through the ResetNotification protocol. + + Register the shutdown notification when mNvmeControllerNumber increased from 0 to 1. +**/ +VOID +NvmeRegisterShutdownNotification ( + VOID + ) +{ + EFI_STATUS Status; + EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify; + + mNvmeControllerNumber++; + if (mNvmeControllerNumber == 1) { + Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **) &ResetNotify); + if (!EFI_ERROR (Status)) { + Status = ResetNotify->RegisterResetNotify (ResetNotify, NvmeShutdownAllControllers); + ASSERT_EFI_ERROR (Status); + } else { + DEBUG ((DEBUG_WARN, "NVME: ResetNotification absent! Shutdown notification cannot be performed!\n")); + } + } +} + +/** + Unregister the shutdown notification through the ResetNotification protocol. + + Unregister the shutdown notification when mNvmeControllerNumber decreased from 1 to 0. +**/ +VOID +NvmeUnregisterShutdownNotification ( + VOID + ) +{ + EFI_STATUS Status; + EFI_RESET_NOTIFICATION_PROTOCOL *ResetNotify; + + mNvmeControllerNumber--; + if (mNvmeControllerNumber == 0) { + Status = gBS->LocateProtocol (&gEfiResetNotificationProtocolGuid, NULL, (VOID **) &ResetNotify); + if (!EFI_ERROR (Status)) { + Status = ResetNotify->UnregisterResetNotify (ResetNotify, NvmeShutdownAllControllers); + ASSERT_EFI_ERROR (Status); + } + } +} |