1 /** @file 2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows 3 NVM Express specification. 4 5 Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR> 6 This program and the accompanying materials 7 are licensed and made available under the terms and conditions of the BSD License 8 which accompanies this distribution. The full text of the license may be found at 9 http://opensource.org/licenses/bsd-license.php. 10 11 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, 12 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 13 14 **/ 15 16 #include "NvmExpress.h" 17 18 /** 19 Read Nvm Express controller capability register. 20 21 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 22 @param Cap The buffer used to store capability register content. 23 24 @return EFI_SUCCESS Successfully read the controller capability register content. 25 @return EFI_DEVICE_ERROR Fail to read the controller capability register. 26 27 **/ 28 EFI_STATUS 29 ReadNvmeControllerCapabilities ( 30 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 31 IN NVME_CAP *Cap 32 ) 33 { 34 EFI_PCI_IO_PROTOCOL *PciIo; 35 EFI_STATUS Status; 36 UINT64 Data; 37 38 PciIo = Private->PciIo; 39 Status = PciIo->Mem.Read ( 40 PciIo, 41 EfiPciIoWidthUint32, 42 NVME_BAR, 43 NVME_CAP_OFFSET, 44 2, 45 &Data 46 ); 47 48 if (EFI_ERROR(Status)) { 49 return Status; 50 } 51 52 WriteUnaligned64 ((UINT64*)Cap, Data); 53 return EFI_SUCCESS; 54 } 55 56 /** 57 Read Nvm Express controller configuration register. 58 59 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 60 @param Cc The buffer used to store configuration register content. 61 62 @return EFI_SUCCESS Successfully read the controller configuration register content. 63 @return EFI_DEVICE_ERROR Fail to read the controller configuration register. 64 65 **/ 66 EFI_STATUS 67 ReadNvmeControllerConfiguration ( 68 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 69 IN NVME_CC *Cc 70 ) 71 { 72 EFI_PCI_IO_PROTOCOL *PciIo; 73 EFI_STATUS Status; 74 UINT32 Data; 75 76 PciIo = Private->PciIo; 77 Status = PciIo->Mem.Read ( 78 PciIo, 79 EfiPciIoWidthUint32, 80 NVME_BAR, 81 NVME_CC_OFFSET, 82 1, 83 &Data 84 ); 85 86 if (EFI_ERROR(Status)) { 87 return Status; 88 } 89 90 WriteUnaligned32 ((UINT32*)Cc, Data); 91 return EFI_SUCCESS; 92 } 93 94 /** 95 Write Nvm Express controller configuration register. 96 97 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 98 @param Cc The buffer used to store the content to be written into configuration register. 99 100 @return EFI_SUCCESS Successfully write data into the controller configuration register. 101 @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register. 102 103 **/ 104 EFI_STATUS 105 WriteNvmeControllerConfiguration ( 106 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 107 IN NVME_CC *Cc 108 ) 109 { 110 EFI_PCI_IO_PROTOCOL *PciIo; 111 EFI_STATUS Status; 112 UINT32 Data; 113 114 PciIo = Private->PciIo; 115 Data = ReadUnaligned32 ((UINT32*)Cc); 116 Status = PciIo->Mem.Write ( 117 PciIo, 118 EfiPciIoWidthUint32, 119 NVME_BAR, 120 NVME_CC_OFFSET, 121 1, 122 &Data 123 ); 124 125 if (EFI_ERROR(Status)) { 126 return Status; 127 } 128 129 DEBUG ((EFI_D_INFO, "Cc.En: %d\n", Cc->En)); 130 DEBUG ((EFI_D_INFO, "Cc.Css: %d\n", Cc->Css)); 131 DEBUG ((EFI_D_INFO, "Cc.Mps: %d\n", Cc->Mps)); 132 DEBUG ((EFI_D_INFO, "Cc.Ams: %d\n", Cc->Ams)); 133 DEBUG ((EFI_D_INFO, "Cc.Shn: %d\n", Cc->Shn)); 134 DEBUG ((EFI_D_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes)); 135 DEBUG ((EFI_D_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes)); 136 137 return EFI_SUCCESS; 138 } 139 140 /** 141 Read Nvm Express controller status register. 142 143 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 144 @param Csts The buffer used to store status register content. 145 146 @return EFI_SUCCESS Successfully read the controller status register content. 147 @return EFI_DEVICE_ERROR Fail to read the controller status register. 148 149 **/ 150 EFI_STATUS 151 ReadNvmeControllerStatus ( 152 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 153 IN NVME_CSTS *Csts 154 ) 155 { 156 EFI_PCI_IO_PROTOCOL *PciIo; 157 EFI_STATUS Status; 158 UINT32 Data; 159 160 PciIo = Private->PciIo; 161 Status = PciIo->Mem.Read ( 162 PciIo, 163 EfiPciIoWidthUint32, 164 NVME_BAR, 165 NVME_CSTS_OFFSET, 166 1, 167 &Data 168 ); 169 170 if (EFI_ERROR(Status)) { 171 return Status; 172 } 173 174 WriteUnaligned32 ((UINT32*)Csts, Data); 175 return EFI_SUCCESS; 176 } 177 178 /** 179 Read Nvm Express admin queue attributes register. 180 181 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 182 @param Aqa The buffer used to store admin queue attributes register content. 183 184 @return EFI_SUCCESS Successfully read the admin queue attributes register content. 185 @return EFI_DEVICE_ERROR Fail to read the admin queue attributes register. 186 187 **/ 188 EFI_STATUS 189 ReadNvmeAdminQueueAttributes ( 190 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 191 IN NVME_AQA *Aqa 192 ) 193 { 194 EFI_PCI_IO_PROTOCOL *PciIo; 195 EFI_STATUS Status; 196 UINT32 Data; 197 198 PciIo = Private->PciIo; 199 Status = PciIo->Mem.Read ( 200 PciIo, 201 EfiPciIoWidthUint32, 202 NVME_BAR, 203 NVME_AQA_OFFSET, 204 1, 205 &Data 206 ); 207 208 if (EFI_ERROR(Status)) { 209 return Status; 210 } 211 212 WriteUnaligned32 ((UINT32*)Aqa, Data); 213 return EFI_SUCCESS; 214 } 215 216 /** 217 Write Nvm Express admin queue attributes register. 218 219 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 220 @param Aqa The buffer used to store the content to be written into admin queue attributes register. 221 222 @return EFI_SUCCESS Successfully write data into the admin queue attributes register. 223 @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register. 224 225 **/ 226 EFI_STATUS 227 WriteNvmeAdminQueueAttributes ( 228 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 229 IN NVME_AQA *Aqa 230 ) 231 { 232 EFI_PCI_IO_PROTOCOL *PciIo; 233 EFI_STATUS Status; 234 UINT32 Data; 235 236 PciIo = Private->PciIo; 237 Data = ReadUnaligned32 ((UINT32*)Aqa); 238 Status = PciIo->Mem.Write ( 239 PciIo, 240 EfiPciIoWidthUint32, 241 NVME_BAR, 242 NVME_AQA_OFFSET, 243 1, 244 &Data 245 ); 246 247 if (EFI_ERROR(Status)) { 248 return Status; 249 } 250 251 DEBUG ((EFI_D_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs)); 252 DEBUG ((EFI_D_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs)); 253 254 return EFI_SUCCESS; 255 } 256 257 /** 258 Read Nvm Express admin submission queue base address register. 259 260 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 261 @param Asq The buffer used to store admin submission queue base address register content. 262 263 @return EFI_SUCCESS Successfully read the admin submission queue base address register content. 264 @return EFI_DEVICE_ERROR Fail to read the admin submission queue base address register. 265 266 **/ 267 EFI_STATUS 268 ReadNvmeAdminSubmissionQueueBaseAddress ( 269 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 270 IN NVME_ASQ *Asq 271 ) 272 { 273 EFI_PCI_IO_PROTOCOL *PciIo; 274 EFI_STATUS Status; 275 UINT64 Data; 276 277 PciIo = Private->PciIo; 278 Status = PciIo->Mem.Read ( 279 PciIo, 280 EfiPciIoWidthUint32, 281 NVME_BAR, 282 NVME_ASQ_OFFSET, 283 2, 284 &Data 285 ); 286 287 if (EFI_ERROR(Status)) { 288 return Status; 289 } 290 291 WriteUnaligned64 ((UINT64*)Asq, Data); 292 return EFI_SUCCESS; 293 } 294 295 /** 296 Write Nvm Express admin submission queue base address register. 297 298 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 299 @param Asq The buffer used to store the content to be written into admin submission queue base address register. 300 301 @return EFI_SUCCESS Successfully write data into the admin submission queue base address register. 302 @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register. 303 304 **/ 305 EFI_STATUS 306 WriteNvmeAdminSubmissionQueueBaseAddress ( 307 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 308 IN NVME_ASQ *Asq 309 ) 310 { 311 EFI_PCI_IO_PROTOCOL *PciIo; 312 EFI_STATUS Status; 313 UINT64 Data; 314 315 PciIo = Private->PciIo; 316 Data = ReadUnaligned64 ((UINT64*)Asq); 317 318 Status = PciIo->Mem.Write ( 319 PciIo, 320 EfiPciIoWidthUint32, 321 NVME_BAR, 322 NVME_ASQ_OFFSET, 323 2, 324 &Data 325 ); 326 327 if (EFI_ERROR(Status)) { 328 return Status; 329 } 330 331 DEBUG ((EFI_D_INFO, "Asq: %lx\n", *Asq)); 332 333 return EFI_SUCCESS; 334 } 335 336 /** 337 Read Nvm Express admin completion queue base address register. 338 339 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 340 @param Acq The buffer used to store admin completion queue base address register content. 341 342 @return EFI_SUCCESS Successfully read the admin completion queue base address register content. 343 @return EFI_DEVICE_ERROR Fail to read the admin completion queue base address register. 344 345 **/ 346 EFI_STATUS 347 ReadNvmeAdminCompletionQueueBaseAddress ( 348 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 349 IN NVME_ACQ *Acq 350 ) 351 { 352 EFI_PCI_IO_PROTOCOL *PciIo; 353 EFI_STATUS Status; 354 UINT64 Data; 355 356 PciIo = Private->PciIo; 357 358 Status = PciIo->Mem.Read ( 359 PciIo, 360 EfiPciIoWidthUint32, 361 NVME_BAR, 362 NVME_ACQ_OFFSET, 363 2, 364 &Data 365 ); 366 367 if (EFI_ERROR(Status)) { 368 return Status; 369 } 370 371 WriteUnaligned64 ((UINT64*)Acq, Data); 372 return EFI_SUCCESS; 373 } 374 375 /** 376 Write Nvm Express admin completion queue base address register. 377 378 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 379 @param Acq The buffer used to store the content to be written into admin completion queue base address register. 380 381 @return EFI_SUCCESS Successfully write data into the admin completion queue base address register. 382 @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register. 383 384 **/ 385 EFI_STATUS 386 WriteNvmeAdminCompletionQueueBaseAddress ( 387 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 388 IN NVME_ACQ *Acq 389 ) 390 { 391 EFI_PCI_IO_PROTOCOL *PciIo; 392 EFI_STATUS Status; 393 UINT64 Data; 394 395 PciIo = Private->PciIo; 396 Data = ReadUnaligned64 ((UINT64*)Acq); 397 398 Status = PciIo->Mem.Write ( 399 PciIo, 400 EfiPciIoWidthUint32, 401 NVME_BAR, 402 NVME_ACQ_OFFSET, 403 2, 404 &Data 405 ); 406 407 if (EFI_ERROR(Status)) { 408 return Status; 409 } 410 411 DEBUG ((EFI_D_INFO, "Acq: %lxh\n", *Acq)); 412 413 return EFI_SUCCESS; 414 } 415 416 /** 417 Disable the Nvm Express controller. 418 419 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 420 421 @return EFI_SUCCESS Successfully disable the controller. 422 @return EFI_DEVICE_ERROR Fail to disable the controller. 423 424 **/ 425 EFI_STATUS 426 NvmeDisableController ( 427 IN NVME_CONTROLLER_PRIVATE_DATA *Private 428 ) 429 { 430 NVME_CC Cc; 431 NVME_CSTS Csts; 432 EFI_STATUS Status; 433 UINT32 Index; 434 UINT8 Timeout; 435 436 // 437 // Read Controller Configuration Register. 438 // 439 Status = ReadNvmeControllerConfiguration (Private, &Cc); 440 if (EFI_ERROR(Status)) { 441 return Status; 442 } 443 444 Cc.En = 0; 445 446 // 447 // Disable the controller. 448 // 449 Status = WriteNvmeControllerConfiguration (Private, &Cc); 450 451 if (EFI_ERROR(Status)) { 452 return Status; 453 } 454 455 // 456 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after 457 // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To. 458 // 459 if (Private->Cap.To == 0) { 460 Timeout = 1; 461 } else { 462 Timeout = Private->Cap.To; 463 } 464 465 for(Index = (Timeout * 500); Index != 0; --Index) { 466 gBS->Stall(1000); 467 468 // 469 // Check if the controller is initialized 470 // 471 Status = ReadNvmeControllerStatus (Private, &Csts); 472 473 if (EFI_ERROR(Status)) { 474 return Status; 475 } 476 477 if (Csts.Rdy == 0) { 478 break; 479 } 480 } 481 482 if (Index == 0) { 483 Status = EFI_DEVICE_ERROR; 484 } 485 486 DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status)); 487 return Status; 488 } 489 490 /** 491 Enable the Nvm Express controller. 492 493 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 494 495 @return EFI_SUCCESS Successfully enable the controller. 496 @return EFI_DEVICE_ERROR Fail to enable the controller. 497 @return EFI_TIMEOUT Fail to enable the controller in given time slot. 498 499 **/ 500 EFI_STATUS 501 NvmeEnableController ( 502 IN NVME_CONTROLLER_PRIVATE_DATA *Private 503 ) 504 { 505 NVME_CC Cc; 506 NVME_CSTS Csts; 507 EFI_STATUS Status; 508 UINT32 Index; 509 UINT8 Timeout; 510 511 // 512 // Enable the controller. 513 // CC.AMS, CC.MPS and CC.CSS are all set to 0. 514 // 515 ZeroMem (&Cc, sizeof (NVME_CC)); 516 Cc.En = 1; 517 Cc.Iosqes = 6; 518 Cc.Iocqes = 4; 519 520 Status = WriteNvmeControllerConfiguration (Private, &Cc); 521 if (EFI_ERROR(Status)) { 522 return Status; 523 } 524 525 // 526 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after 527 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To. 528 // 529 if (Private->Cap.To == 0) { 530 Timeout = 1; 531 } else { 532 Timeout = Private->Cap.To; 533 } 534 535 for(Index = (Timeout * 500); Index != 0; --Index) { 536 gBS->Stall(1000); 537 538 // 539 // Check if the controller is initialized 540 // 541 Status = ReadNvmeControllerStatus (Private, &Csts); 542 543 if (EFI_ERROR(Status)) { 544 return Status; 545 } 546 547 if (Csts.Rdy) { 548 break; 549 } 550 } 551 552 if (Index == 0) { 553 Status = EFI_TIMEOUT; 554 } 555 556 DEBUG ((EFI_D_INFO, "NVMe controller is enabled with status [%r].\n", Status)); 557 return Status; 558 } 559 560 /** 561 Get identify controller data. 562 563 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 564 @param Buffer The buffer used to store the identify controller data. 565 566 @return EFI_SUCCESS Successfully get the identify controller data. 567 @return EFI_DEVICE_ERROR Fail to get the identify controller data. 568 569 **/ 570 EFI_STATUS 571 NvmeIdentifyController ( 572 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 573 IN VOID *Buffer 574 ) 575 { 576 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; 577 EFI_NVM_EXPRESS_COMMAND Command; 578 EFI_NVM_EXPRESS_COMPLETION Completion; 579 EFI_STATUS Status; 580 581 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); 582 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); 583 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); 584 585 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD; 586 // 587 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h. 588 // For the Identify command, the Namespace Identifier is only used for the Namespace data structure. 589 // 590 Command.Nsid = 0; 591 592 CommandPacket.NvmeCmd = &Command; 593 CommandPacket.NvmeCompletion = &Completion; 594 CommandPacket.TransferBuffer = Buffer; 595 CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA); 596 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; 597 CommandPacket.QueueType = NVME_ADMIN_QUEUE; 598 // 599 // Set bit 0 (Cns bit) to 1 to identify a controller 600 // 601 Command.Cdw10 = 1; 602 Command.Flags = CDW10_VALID; 603 604 Status = Private->Passthru.PassThru ( 605 &Private->Passthru, 606 NVME_CONTROLLER_ID, 607 &CommandPacket, 608 NULL 609 ); 610 611 return Status; 612 } 613 614 /** 615 Get specified identify namespace data. 616 617 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 618 @param NamespaceId The specified namespace identifier. 619 @param Buffer The buffer used to store the identify namespace data. 620 621 @return EFI_SUCCESS Successfully get the identify namespace data. 622 @return EFI_DEVICE_ERROR Fail to get the identify namespace data. 623 624 **/ 625 EFI_STATUS 626 NvmeIdentifyNamespace ( 627 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 628 IN UINT32 NamespaceId, 629 IN VOID *Buffer 630 ) 631 { 632 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; 633 EFI_NVM_EXPRESS_COMMAND Command; 634 EFI_NVM_EXPRESS_COMPLETION Completion; 635 EFI_STATUS Status; 636 637 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); 638 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); 639 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); 640 641 CommandPacket.NvmeCmd = &Command; 642 CommandPacket.NvmeCompletion = &Completion; 643 644 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD; 645 Command.Nsid = NamespaceId; 646 CommandPacket.TransferBuffer = Buffer; 647 CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA); 648 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; 649 CommandPacket.QueueType = NVME_ADMIN_QUEUE; 650 // 651 // Set bit 0 (Cns bit) to 1 to identify a namespace 652 // 653 CommandPacket.NvmeCmd->Cdw10 = 0; 654 CommandPacket.NvmeCmd->Flags = CDW10_VALID; 655 656 Status = Private->Passthru.PassThru ( 657 &Private->Passthru, 658 NamespaceId, 659 &CommandPacket, 660 NULL 661 ); 662 663 return Status; 664 } 665 666 /** 667 Create io completion queue. 668 669 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 670 671 @return EFI_SUCCESS Successfully create io completion queue. 672 @return EFI_DEVICE_ERROR Fail to create io completion queue. 673 674 **/ 675 EFI_STATUS 676 NvmeCreateIoCompletionQueue ( 677 IN NVME_CONTROLLER_PRIVATE_DATA *Private 678 ) 679 { 680 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; 681 EFI_NVM_EXPRESS_COMMAND Command; 682 EFI_NVM_EXPRESS_COMPLETION Completion; 683 EFI_STATUS Status; 684 NVME_ADMIN_CRIOCQ CrIoCq; 685 UINT32 Index; 686 UINT16 QueueSize; 687 688 Status = EFI_SUCCESS; 689 690 for (Index = 1; Index < NVME_MAX_QUEUES; Index++) { 691 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); 692 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); 693 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); 694 ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ)); 695 696 CommandPacket.NvmeCmd = &Command; 697 CommandPacket.NvmeCompletion = &Completion; 698 699 Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD; 700 CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index]; 701 CommandPacket.TransferLength = EFI_PAGE_SIZE; 702 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; 703 CommandPacket.QueueType = NVME_ADMIN_QUEUE; 704 705 if (Index == 1) { 706 QueueSize = NVME_CCQ_SIZE; 707 } else { 708 if (Private->Cap.Mqes > NVME_ASYNC_CCQ_SIZE) { 709 QueueSize = NVME_ASYNC_CCQ_SIZE; 710 } else { 711 QueueSize = Private->Cap.Mqes; 712 } 713 } 714 715 CrIoCq.Qid = Index; 716 CrIoCq.Qsize = QueueSize; 717 CrIoCq.Pc = 1; 718 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ)); 719 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID; 720 721 Status = Private->Passthru.PassThru ( 722 &Private->Passthru, 723 0, 724 &CommandPacket, 725 NULL 726 ); 727 if (EFI_ERROR (Status)) { 728 break; 729 } 730 } 731 732 return Status; 733 } 734 735 /** 736 Create io submission queue. 737 738 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 739 740 @return EFI_SUCCESS Successfully create io submission queue. 741 @return EFI_DEVICE_ERROR Fail to create io submission queue. 742 743 **/ 744 EFI_STATUS 745 NvmeCreateIoSubmissionQueue ( 746 IN NVME_CONTROLLER_PRIVATE_DATA *Private 747 ) 748 { 749 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; 750 EFI_NVM_EXPRESS_COMMAND Command; 751 EFI_NVM_EXPRESS_COMPLETION Completion; 752 EFI_STATUS Status; 753 NVME_ADMIN_CRIOSQ CrIoSq; 754 UINT32 Index; 755 UINT16 QueueSize; 756 757 Status = EFI_SUCCESS; 758 759 for (Index = 1; Index < NVME_MAX_QUEUES; Index++) { 760 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); 761 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); 762 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); 763 ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ)); 764 765 CommandPacket.NvmeCmd = &Command; 766 CommandPacket.NvmeCompletion = &Completion; 767 768 Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD; 769 CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index]; 770 CommandPacket.TransferLength = EFI_PAGE_SIZE; 771 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; 772 CommandPacket.QueueType = NVME_ADMIN_QUEUE; 773 774 if (Index == 1) { 775 QueueSize = NVME_CSQ_SIZE; 776 } else { 777 if (Private->Cap.Mqes > NVME_ASYNC_CSQ_SIZE) { 778 QueueSize = NVME_ASYNC_CSQ_SIZE; 779 } else { 780 QueueSize = Private->Cap.Mqes; 781 } 782 } 783 784 CrIoSq.Qid = Index; 785 CrIoSq.Qsize = QueueSize; 786 CrIoSq.Pc = 1; 787 CrIoSq.Cqid = Index; 788 CrIoSq.Qprio = 0; 789 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ)); 790 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID; 791 792 Status = Private->Passthru.PassThru ( 793 &Private->Passthru, 794 0, 795 &CommandPacket, 796 NULL 797 ); 798 if (EFI_ERROR (Status)) { 799 break; 800 } 801 } 802 803 return Status; 804 } 805 806 /** 807 Initialize the Nvm Express controller. 808 809 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 810 811 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully. 812 @retval Others A device error occurred while initializing the controller. 813 814 **/ 815 EFI_STATUS 816 NvmeControllerInit ( 817 IN NVME_CONTROLLER_PRIVATE_DATA *Private 818 ) 819 { 820 EFI_STATUS Status; 821 EFI_PCI_IO_PROTOCOL *PciIo; 822 UINT64 Supports; 823 NVME_AQA Aqa; 824 NVME_ASQ Asq; 825 NVME_ACQ Acq; 826 UINT8 Sn[21]; 827 UINT8 Mn[41]; 828 // 829 // Save original PCI attributes and enable this controller. 830 // 831 PciIo = Private->PciIo; 832 Status = PciIo->Attributes ( 833 PciIo, 834 EfiPciIoAttributeOperationGet, 835 0, 836 &Private->PciAttributes 837 ); 838 839 if (EFI_ERROR (Status)) { 840 return Status; 841 } 842 843 Status = PciIo->Attributes ( 844 PciIo, 845 EfiPciIoAttributeOperationSupported, 846 0, 847 &Supports 848 ); 849 850 if (!EFI_ERROR (Status)) { 851 Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE; 852 Status = PciIo->Attributes ( 853 PciIo, 854 EfiPciIoAttributeOperationEnable, 855 Supports, 856 NULL 857 ); 858 } 859 860 if (EFI_ERROR (Status)) { 861 DEBUG ((EFI_D_INFO, "NvmeControllerInit: failed to enable controller\n")); 862 return Status; 863 } 864 865 // 866 // Enable 64-bit DMA support in the PCI layer. 867 // 868 Status = PciIo->Attributes ( 869 PciIo, 870 EfiPciIoAttributeOperationEnable, 871 EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE, 872 NULL 873 ); 874 if (EFI_ERROR (Status)) { 875 DEBUG ((EFI_D_WARN, "NvmeControllerInit: failed to enable 64-bit DMA (%r)\n", Status)); 876 } 877 878 // 879 // Read the Controller Capabilities register and verify that the NVM command set is supported 880 // 881 Status = ReadNvmeControllerCapabilities (Private, &Private->Cap); 882 if (EFI_ERROR (Status)) { 883 return Status; 884 } 885 886 if (Private->Cap.Css != 0x01) { 887 DEBUG ((EFI_D_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n")); 888 return EFI_UNSUPPORTED; 889 } 890 891 // 892 // Currently the driver only supports 4k page size. 893 // 894 ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT); 895 896 Private->Cid[0] = 0; 897 Private->Cid[1] = 0; 898 Private->Cid[2] = 0; 899 Private->Pt[0] = 0; 900 Private->Pt[1] = 0; 901 Private->Pt[2] = 0; 902 Private->SqTdbl[0].Sqt = 0; 903 Private->SqTdbl[1].Sqt = 0; 904 Private->SqTdbl[2].Sqt = 0; 905 Private->CqHdbl[0].Cqh = 0; 906 Private->CqHdbl[1].Cqh = 0; 907 Private->CqHdbl[2].Cqh = 0; 908 Private->AsyncSqHead = 0; 909 910 Status = NvmeDisableController (Private); 911 912 if (EFI_ERROR(Status)) { 913 return Status; 914 } 915 916 // 917 // set number of entries admin submission & completion queues. 918 // 919 Aqa.Asqs = NVME_ASQ_SIZE; 920 Aqa.Rsvd1 = 0; 921 Aqa.Acqs = NVME_ACQ_SIZE; 922 Aqa.Rsvd2 = 0; 923 924 // 925 // Address of admin submission queue. 926 // 927 Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF; 928 929 // 930 // Address of admin completion queue. 931 // 932 Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF; 933 934 // 935 // Address of I/O submission & completion queue. 936 // 937 ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6)); 938 Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); 939 Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); 940 Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE); 941 Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE); 942 Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE); 943 Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE); 944 Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE); 945 Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE); 946 Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE); 947 Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE); 948 Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE); 949 Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE); 950 951 DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer)); 952 DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs)); 953 DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs)); 954 DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0])); 955 DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0])); 956 DEBUG ((EFI_D_INFO, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1])); 957 DEBUG ((EFI_D_INFO, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1])); 958 DEBUG ((EFI_D_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2])); 959 DEBUG ((EFI_D_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2])); 960 961 // 962 // Program admin queue attributes. 963 // 964 Status = WriteNvmeAdminQueueAttributes (Private, &Aqa); 965 966 if (EFI_ERROR(Status)) { 967 return Status; 968 } 969 970 // 971 // Program admin submission queue address. 972 // 973 Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq); 974 975 if (EFI_ERROR(Status)) { 976 return Status; 977 } 978 979 // 980 // Program admin completion queue address. 981 // 982 Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq); 983 984 if (EFI_ERROR(Status)) { 985 return Status; 986 } 987 988 Status = NvmeEnableController (Private); 989 if (EFI_ERROR(Status)) { 990 return Status; 991 } 992 993 // 994 // Allocate buffer for Identify Controller data 995 // 996 if (Private->ControllerData == NULL) { 997 Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA)); 998 999 if (Private->ControllerData == NULL) { 1000 return EFI_OUT_OF_RESOURCES; 1001 } 1002 } 1003 1004 // 1005 // Get current Identify Controller Data 1006 // 1007 Status = NvmeIdentifyController (Private, Private->ControllerData); 1008 1009 if (EFI_ERROR(Status)) { 1010 FreePool(Private->ControllerData); 1011 Private->ControllerData = NULL; 1012 return EFI_NOT_FOUND; 1013 } 1014 1015 // 1016 // Dump NvmExpress Identify Controller Data 1017 // 1018 CopyMem (Sn, Private->ControllerData->Sn, sizeof (Private->ControllerData->Sn)); 1019 Sn[20] = 0; 1020 CopyMem (Mn, Private->ControllerData->Mn, sizeof (Private->ControllerData->Mn)); 1021 Mn[40] = 0; 1022 DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n")); 1023 DEBUG ((EFI_D_INFO, " PCI VID : 0x%x\n", Private->ControllerData->Vid)); 1024 DEBUG ((EFI_D_INFO, " PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid)); 1025 DEBUG ((EFI_D_INFO, " SN : %a\n", Sn)); 1026 DEBUG ((EFI_D_INFO, " MN : %a\n", Mn)); 1027 DEBUG ((EFI_D_INFO, " FR : 0x%x\n", *((UINT64*)Private->ControllerData->Fr))); 1028 DEBUG ((EFI_D_INFO, " RAB : 0x%x\n", Private->ControllerData->Rab)); 1029 DEBUG ((EFI_D_INFO, " IEEE : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oui)); 1030 DEBUG ((EFI_D_INFO, " AERL : 0x%x\n", Private->ControllerData->Aerl)); 1031 DEBUG ((EFI_D_INFO, " SQES : 0x%x\n", Private->ControllerData->Sqes)); 1032 DEBUG ((EFI_D_INFO, " CQES : 0x%x\n", Private->ControllerData->Cqes)); 1033 DEBUG ((EFI_D_INFO, " NN : 0x%x\n", Private->ControllerData->Nn)); 1034 1035 // 1036 // Create two I/O completion queues. 1037 // One for blocking I/O, one for non-blocking I/O. 1038 // 1039 Status = NvmeCreateIoCompletionQueue (Private); 1040 if (EFI_ERROR(Status)) { 1041 return Status; 1042 } 1043 1044 // 1045 // Create two I/O Submission queues. 1046 // One for blocking I/O, one for non-blocking I/O. 1047 // 1048 Status = NvmeCreateIoSubmissionQueue (Private); 1049 if (EFI_ERROR(Status)) { 1050 return Status; 1051 } 1052 1053 return Status; 1054 } 1055 1056