1 /** @file 2 NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows 3 NVM Express specification. 4 5 Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR> 6 This program and the accompanying materials 7 are licensed and made available under the terms and conditions of the BSD License 8 which accompanies this distribution. The full text of the license may be found at 9 http://opensource.org/licenses/bsd-license.php. 10 11 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, 12 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 13 14 **/ 15 16 #include "NvmExpress.h" 17 18 /** 19 Read Nvm Express controller capability register. 20 21 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 22 @param Cap The buffer used to store capability register content. 23 24 @return EFI_SUCCESS Successfully read the controller capability register content. 25 @return EFI_DEVICE_ERROR Fail to read the controller capability register. 26 27 **/ 28 EFI_STATUS 29 ReadNvmeControllerCapabilities ( 30 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 31 IN NVME_CAP *Cap 32 ) 33 { 34 EFI_PCI_IO_PROTOCOL *PciIo; 35 EFI_STATUS Status; 36 UINT64 Data; 37 38 PciIo = Private->PciIo; 39 Status = PciIo->Mem.Read ( 40 PciIo, 41 EfiPciIoWidthUint32, 42 NVME_BAR, 43 NVME_CAP_OFFSET, 44 2, 45 &Data 46 ); 47 48 if (EFI_ERROR(Status)) { 49 return Status; 50 } 51 52 WriteUnaligned64 ((UINT64*)Cap, Data); 53 return EFI_SUCCESS; 54 } 55 56 /** 57 Read Nvm Express controller configuration register. 58 59 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 60 @param Cc The buffer used to store configuration register content. 61 62 @return EFI_SUCCESS Successfully read the controller configuration register content. 63 @return EFI_DEVICE_ERROR Fail to read the controller configuration register. 64 65 **/ 66 EFI_STATUS 67 ReadNvmeControllerConfiguration ( 68 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 69 IN NVME_CC *Cc 70 ) 71 { 72 EFI_PCI_IO_PROTOCOL *PciIo; 73 EFI_STATUS Status; 74 UINT32 Data; 75 76 PciIo = Private->PciIo; 77 Status = PciIo->Mem.Read ( 78 PciIo, 79 EfiPciIoWidthUint32, 80 NVME_BAR, 81 NVME_CC_OFFSET, 82 1, 83 &Data 84 ); 85 86 if (EFI_ERROR(Status)) { 87 return Status; 88 } 89 90 WriteUnaligned32 ((UINT32*)Cc, Data); 91 return EFI_SUCCESS; 92 } 93 94 /** 95 Write Nvm Express controller configuration register. 96 97 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 98 @param Cc The buffer used to store the content to be written into configuration register. 99 100 @return EFI_SUCCESS Successfully write data into the controller configuration register. 101 @return EFI_DEVICE_ERROR Fail to write data into the controller configuration register. 102 103 **/ 104 EFI_STATUS 105 WriteNvmeControllerConfiguration ( 106 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 107 IN NVME_CC *Cc 108 ) 109 { 110 EFI_PCI_IO_PROTOCOL *PciIo; 111 EFI_STATUS Status; 112 UINT32 Data; 113 114 PciIo = Private->PciIo; 115 Data = ReadUnaligned32 ((UINT32*)Cc); 116 Status = PciIo->Mem.Write ( 117 PciIo, 118 EfiPciIoWidthUint32, 119 NVME_BAR, 120 NVME_CC_OFFSET, 121 1, 122 &Data 123 ); 124 125 if (EFI_ERROR(Status)) { 126 return Status; 127 } 128 129 DEBUG ((EFI_D_INFO, "Cc.En: %d\n", Cc->En)); 130 DEBUG ((EFI_D_INFO, "Cc.Css: %d\n", Cc->Css)); 131 DEBUG ((EFI_D_INFO, "Cc.Mps: %d\n", Cc->Mps)); 132 DEBUG ((EFI_D_INFO, "Cc.Ams: %d\n", Cc->Ams)); 133 DEBUG ((EFI_D_INFO, "Cc.Shn: %d\n", Cc->Shn)); 134 DEBUG ((EFI_D_INFO, "Cc.Iosqes: %d\n", Cc->Iosqes)); 135 DEBUG ((EFI_D_INFO, "Cc.Iocqes: %d\n", Cc->Iocqes)); 136 137 return EFI_SUCCESS; 138 } 139 140 /** 141 Read Nvm Express controller status register. 142 143 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 144 @param Csts The buffer used to store status register content. 145 146 @return EFI_SUCCESS Successfully read the controller status register content. 147 @return EFI_DEVICE_ERROR Fail to read the controller status register. 148 149 **/ 150 EFI_STATUS 151 ReadNvmeControllerStatus ( 152 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 153 IN NVME_CSTS *Csts 154 ) 155 { 156 EFI_PCI_IO_PROTOCOL *PciIo; 157 EFI_STATUS Status; 158 UINT32 Data; 159 160 PciIo = Private->PciIo; 161 Status = PciIo->Mem.Read ( 162 PciIo, 163 EfiPciIoWidthUint32, 164 NVME_BAR, 165 NVME_CSTS_OFFSET, 166 1, 167 &Data 168 ); 169 170 if (EFI_ERROR(Status)) { 171 return Status; 172 } 173 174 WriteUnaligned32 ((UINT32*)Csts, Data); 175 return EFI_SUCCESS; 176 } 177 178 /** 179 Read Nvm Express admin queue attributes register. 180 181 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 182 @param Aqa The buffer used to store admin queue attributes register content. 183 184 @return EFI_SUCCESS Successfully read the admin queue attributes register content. 185 @return EFI_DEVICE_ERROR Fail to read the admin queue attributes register. 186 187 **/ 188 EFI_STATUS 189 ReadNvmeAdminQueueAttributes ( 190 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 191 IN NVME_AQA *Aqa 192 ) 193 { 194 EFI_PCI_IO_PROTOCOL *PciIo; 195 EFI_STATUS Status; 196 UINT32 Data; 197 198 PciIo = Private->PciIo; 199 Status = PciIo->Mem.Read ( 200 PciIo, 201 EfiPciIoWidthUint32, 202 NVME_BAR, 203 NVME_AQA_OFFSET, 204 1, 205 &Data 206 ); 207 208 if (EFI_ERROR(Status)) { 209 return Status; 210 } 211 212 WriteUnaligned32 ((UINT32*)Aqa, Data); 213 return EFI_SUCCESS; 214 } 215 216 /** 217 Write Nvm Express admin queue attributes register. 218 219 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 220 @param Aqa The buffer used to store the content to be written into admin queue attributes register. 221 222 @return EFI_SUCCESS Successfully write data into the admin queue attributes register. 223 @return EFI_DEVICE_ERROR Fail to write data into the admin queue attributes register. 224 225 **/ 226 EFI_STATUS 227 WriteNvmeAdminQueueAttributes ( 228 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 229 IN NVME_AQA *Aqa 230 ) 231 { 232 EFI_PCI_IO_PROTOCOL *PciIo; 233 EFI_STATUS Status; 234 UINT32 Data; 235 236 PciIo = Private->PciIo; 237 Data = ReadUnaligned32 ((UINT32*)Aqa); 238 Status = PciIo->Mem.Write ( 239 PciIo, 240 EfiPciIoWidthUint32, 241 NVME_BAR, 242 NVME_AQA_OFFSET, 243 1, 244 &Data 245 ); 246 247 if (EFI_ERROR(Status)) { 248 return Status; 249 } 250 251 DEBUG ((EFI_D_INFO, "Aqa.Asqs: %d\n", Aqa->Asqs)); 252 DEBUG ((EFI_D_INFO, "Aqa.Acqs: %d\n", Aqa->Acqs)); 253 254 return EFI_SUCCESS; 255 } 256 257 /** 258 Read Nvm Express admin submission queue base address register. 259 260 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 261 @param Asq The buffer used to store admin submission queue base address register content. 262 263 @return EFI_SUCCESS Successfully read the admin submission queue base address register content. 264 @return EFI_DEVICE_ERROR Fail to read the admin submission queue base address register. 265 266 **/ 267 EFI_STATUS 268 ReadNvmeAdminSubmissionQueueBaseAddress ( 269 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 270 IN NVME_ASQ *Asq 271 ) 272 { 273 EFI_PCI_IO_PROTOCOL *PciIo; 274 EFI_STATUS Status; 275 UINT64 Data; 276 277 PciIo = Private->PciIo; 278 Status = PciIo->Mem.Read ( 279 PciIo, 280 EfiPciIoWidthUint32, 281 NVME_BAR, 282 NVME_ASQ_OFFSET, 283 2, 284 &Data 285 ); 286 287 if (EFI_ERROR(Status)) { 288 return Status; 289 } 290 291 WriteUnaligned64 ((UINT64*)Asq, Data); 292 return EFI_SUCCESS; 293 } 294 295 /** 296 Write Nvm Express admin submission queue base address register. 297 298 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 299 @param Asq The buffer used to store the content to be written into admin submission queue base address register. 300 301 @return EFI_SUCCESS Successfully write data into the admin submission queue base address register. 302 @return EFI_DEVICE_ERROR Fail to write data into the admin submission queue base address register. 303 304 **/ 305 EFI_STATUS 306 WriteNvmeAdminSubmissionQueueBaseAddress ( 307 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 308 IN NVME_ASQ *Asq 309 ) 310 { 311 EFI_PCI_IO_PROTOCOL *PciIo; 312 EFI_STATUS Status; 313 UINT64 Data; 314 315 PciIo = Private->PciIo; 316 Data = ReadUnaligned64 ((UINT64*)Asq); 317 318 Status = PciIo->Mem.Write ( 319 PciIo, 320 EfiPciIoWidthUint32, 321 NVME_BAR, 322 NVME_ASQ_OFFSET, 323 2, 324 &Data 325 ); 326 327 if (EFI_ERROR(Status)) { 328 return Status; 329 } 330 331 DEBUG ((EFI_D_INFO, "Asq: %lx\n", *Asq)); 332 333 return EFI_SUCCESS; 334 } 335 336 /** 337 Read Nvm Express admin completion queue base address register. 338 339 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 340 @param Acq The buffer used to store admin completion queue base address register content. 341 342 @return EFI_SUCCESS Successfully read the admin completion queue base address register content. 343 @return EFI_DEVICE_ERROR Fail to read the admin completion queue base address register. 344 345 **/ 346 EFI_STATUS 347 ReadNvmeAdminCompletionQueueBaseAddress ( 348 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 349 IN NVME_ACQ *Acq 350 ) 351 { 352 EFI_PCI_IO_PROTOCOL *PciIo; 353 EFI_STATUS Status; 354 UINT64 Data; 355 356 PciIo = Private->PciIo; 357 358 Status = PciIo->Mem.Read ( 359 PciIo, 360 EfiPciIoWidthUint32, 361 NVME_BAR, 362 NVME_ACQ_OFFSET, 363 2, 364 &Data 365 ); 366 367 if (EFI_ERROR(Status)) { 368 return Status; 369 } 370 371 WriteUnaligned64 ((UINT64*)Acq, Data); 372 return EFI_SUCCESS; 373 } 374 375 /** 376 Write Nvm Express admin completion queue base address register. 377 378 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 379 @param Acq The buffer used to store the content to be written into admin completion queue base address register. 380 381 @return EFI_SUCCESS Successfully write data into the admin completion queue base address register. 382 @return EFI_DEVICE_ERROR Fail to write data into the admin completion queue base address register. 383 384 **/ 385 EFI_STATUS 386 WriteNvmeAdminCompletionQueueBaseAddress ( 387 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 388 IN NVME_ACQ *Acq 389 ) 390 { 391 EFI_PCI_IO_PROTOCOL *PciIo; 392 EFI_STATUS Status; 393 UINT64 Data; 394 395 PciIo = Private->PciIo; 396 Data = ReadUnaligned64 ((UINT64*)Acq); 397 398 Status = PciIo->Mem.Write ( 399 PciIo, 400 EfiPciIoWidthUint32, 401 NVME_BAR, 402 NVME_ACQ_OFFSET, 403 2, 404 &Data 405 ); 406 407 if (EFI_ERROR(Status)) { 408 return Status; 409 } 410 411 DEBUG ((EFI_D_INFO, "Acq: %lxh\n", *Acq)); 412 413 return EFI_SUCCESS; 414 } 415 416 /** 417 Disable the Nvm Express controller. 418 419 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 420 421 @return EFI_SUCCESS Successfully disable the controller. 422 @return EFI_DEVICE_ERROR Fail to disable the controller. 423 424 **/ 425 EFI_STATUS 426 NvmeDisableController ( 427 IN NVME_CONTROLLER_PRIVATE_DATA *Private 428 ) 429 { 430 NVME_CC Cc; 431 NVME_CSTS Csts; 432 EFI_STATUS Status; 433 UINT32 Index; 434 UINT8 Timeout; 435 436 // 437 // Read Controller Configuration Register. 438 // 439 Status = ReadNvmeControllerConfiguration (Private, &Cc); 440 if (EFI_ERROR(Status)) { 441 return Status; 442 } 443 444 Cc.En = 0; 445 446 // 447 // Disable the controller. 448 // 449 Status = WriteNvmeControllerConfiguration (Private, &Cc); 450 451 if (EFI_ERROR(Status)) { 452 return Status; 453 } 454 455 // 456 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to transition from 1 to 0 after 457 // Cc.Enable transition from 1 to 0. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To. 458 // 459 if (Private->Cap.To == 0) { 460 Timeout = 1; 461 } else { 462 Timeout = Private->Cap.To; 463 } 464 465 for(Index = (Timeout * 500); Index != 0; --Index) { 466 gBS->Stall(1000); 467 468 // 469 // Check if the controller is initialized 470 // 471 Status = ReadNvmeControllerStatus (Private, &Csts); 472 473 if (EFI_ERROR(Status)) { 474 return Status; 475 } 476 477 if (Csts.Rdy == 0) { 478 break; 479 } 480 } 481 482 if (Index == 0) { 483 Status = EFI_DEVICE_ERROR; 484 } 485 486 DEBUG ((EFI_D_INFO, "NVMe controller is disabled with status [%r].\n", Status)); 487 return Status; 488 } 489 490 /** 491 Enable the Nvm Express controller. 492 493 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 494 495 @return EFI_SUCCESS Successfully enable the controller. 496 @return EFI_DEVICE_ERROR Fail to enable the controller. 497 @return EFI_TIMEOUT Fail to enable the controller in given time slot. 498 499 **/ 500 EFI_STATUS 501 NvmeEnableController ( 502 IN NVME_CONTROLLER_PRIVATE_DATA *Private 503 ) 504 { 505 NVME_CC Cc; 506 NVME_CSTS Csts; 507 EFI_STATUS Status; 508 UINT32 Index; 509 UINT8 Timeout; 510 511 // 512 // Enable the controller. 513 // CC.AMS, CC.MPS and CC.CSS are all set to 0. 514 // 515 ZeroMem (&Cc, sizeof (NVME_CC)); 516 Cc.En = 1; 517 Cc.Iosqes = 6; 518 Cc.Iocqes = 4; 519 520 Status = WriteNvmeControllerConfiguration (Private, &Cc); 521 if (EFI_ERROR(Status)) { 522 return Status; 523 } 524 525 // 526 // Cap.To specifies max delay time in 500ms increments for Csts.Rdy to set after 527 // Cc.Enable. Loop produces a 1 millisecond delay per itteration, up to 500 * Cap.To. 528 // 529 if (Private->Cap.To == 0) { 530 Timeout = 1; 531 } else { 532 Timeout = Private->Cap.To; 533 } 534 535 for(Index = (Timeout * 500); Index != 0; --Index) { 536 gBS->Stall(1000); 537 538 // 539 // Check if the controller is initialized 540 // 541 Status = ReadNvmeControllerStatus (Private, &Csts); 542 543 if (EFI_ERROR(Status)) { 544 return Status; 545 } 546 547 if (Csts.Rdy) { 548 break; 549 } 550 } 551 552 if (Index == 0) { 553 Status = EFI_TIMEOUT; 554 } 555 556 DEBUG ((EFI_D_INFO, "NVMe controller is enabled with status [%r].\n", Status)); 557 return Status; 558 } 559 560 /** 561 Get identify controller data. 562 563 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 564 @param Buffer The buffer used to store the identify controller data. 565 566 @return EFI_SUCCESS Successfully get the identify controller data. 567 @return EFI_DEVICE_ERROR Fail to get the identify controller data. 568 569 **/ 570 EFI_STATUS 571 NvmeIdentifyController ( 572 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 573 IN VOID *Buffer 574 ) 575 { 576 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; 577 EFI_NVM_EXPRESS_COMMAND Command; 578 EFI_NVM_EXPRESS_COMPLETION Completion; 579 EFI_STATUS Status; 580 581 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); 582 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); 583 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); 584 585 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD; 586 // 587 // According to Nvm Express 1.1 spec Figure 38, When not used, the field shall be cleared to 0h. 588 // For the Identify command, the Namespace Identifier is only used for the Namespace data structure. 589 // 590 Command.Nsid = 0; 591 592 CommandPacket.NvmeCmd = &Command; 593 CommandPacket.NvmeCompletion = &Completion; 594 CommandPacket.TransferBuffer = Buffer; 595 CommandPacket.TransferLength = sizeof (NVME_ADMIN_CONTROLLER_DATA); 596 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; 597 CommandPacket.QueueType = NVME_ADMIN_QUEUE; 598 // 599 // Set bit 0 (Cns bit) to 1 to identify a controller 600 // 601 Command.Cdw10 = 1; 602 Command.Flags = CDW10_VALID; 603 604 Status = Private->Passthru.PassThru ( 605 &Private->Passthru, 606 NVME_CONTROLLER_ID, 607 &CommandPacket, 608 NULL 609 ); 610 611 return Status; 612 } 613 614 /** 615 Get specified identify namespace data. 616 617 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 618 @param NamespaceId The specified namespace identifier. 619 @param Buffer The buffer used to store the identify namespace data. 620 621 @return EFI_SUCCESS Successfully get the identify namespace data. 622 @return EFI_DEVICE_ERROR Fail to get the identify namespace data. 623 624 **/ 625 EFI_STATUS 626 NvmeIdentifyNamespace ( 627 IN NVME_CONTROLLER_PRIVATE_DATA *Private, 628 IN UINT32 NamespaceId, 629 IN VOID *Buffer 630 ) 631 { 632 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; 633 EFI_NVM_EXPRESS_COMMAND Command; 634 EFI_NVM_EXPRESS_COMPLETION Completion; 635 EFI_STATUS Status; 636 637 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); 638 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); 639 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); 640 641 CommandPacket.NvmeCmd = &Command; 642 CommandPacket.NvmeCompletion = &Completion; 643 644 Command.Cdw0.Opcode = NVME_ADMIN_IDENTIFY_CMD; 645 Command.Nsid = NamespaceId; 646 CommandPacket.TransferBuffer = Buffer; 647 CommandPacket.TransferLength = sizeof (NVME_ADMIN_NAMESPACE_DATA); 648 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; 649 CommandPacket.QueueType = NVME_ADMIN_QUEUE; 650 // 651 // Set bit 0 (Cns bit) to 1 to identify a namespace 652 // 653 CommandPacket.NvmeCmd->Cdw10 = 0; 654 CommandPacket.NvmeCmd->Flags = CDW10_VALID; 655 656 Status = Private->Passthru.PassThru ( 657 &Private->Passthru, 658 NamespaceId, 659 &CommandPacket, 660 NULL 661 ); 662 663 return Status; 664 } 665 666 /** 667 Create io completion queue. 668 669 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 670 671 @return EFI_SUCCESS Successfully create io completion queue. 672 @return EFI_DEVICE_ERROR Fail to create io completion queue. 673 674 **/ 675 EFI_STATUS 676 NvmeCreateIoCompletionQueue ( 677 IN NVME_CONTROLLER_PRIVATE_DATA *Private 678 ) 679 { 680 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; 681 EFI_NVM_EXPRESS_COMMAND Command; 682 EFI_NVM_EXPRESS_COMPLETION Completion; 683 EFI_STATUS Status; 684 NVME_ADMIN_CRIOCQ CrIoCq; 685 686 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); 687 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); 688 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); 689 ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ)); 690 691 CommandPacket.NvmeCmd = &Command; 692 CommandPacket.NvmeCompletion = &Completion; 693 694 Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD; 695 CommandPacket.TransferBuffer = Private->CqBufferPciAddr[1]; 696 CommandPacket.TransferLength = EFI_PAGE_SIZE; 697 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; 698 CommandPacket.QueueType = NVME_ADMIN_QUEUE; 699 700 CrIoCq.Qid = NVME_IO_QUEUE; 701 CrIoCq.Qsize = NVME_CCQ_SIZE; 702 CrIoCq.Pc = 1; 703 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ)); 704 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID; 705 706 Status = Private->Passthru.PassThru ( 707 &Private->Passthru, 708 0, 709 &CommandPacket, 710 NULL 711 ); 712 713 return Status; 714 } 715 716 /** 717 Create io submission queue. 718 719 @param Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 720 721 @return EFI_SUCCESS Successfully create io submission queue. 722 @return EFI_DEVICE_ERROR Fail to create io submission queue. 723 724 **/ 725 EFI_STATUS 726 NvmeCreateIoSubmissionQueue ( 727 IN NVME_CONTROLLER_PRIVATE_DATA *Private 728 ) 729 { 730 EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET CommandPacket; 731 EFI_NVM_EXPRESS_COMMAND Command; 732 EFI_NVM_EXPRESS_COMPLETION Completion; 733 EFI_STATUS Status; 734 NVME_ADMIN_CRIOSQ CrIoSq; 735 736 ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET)); 737 ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND)); 738 ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION)); 739 ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ)); 740 741 CommandPacket.NvmeCmd = &Command; 742 CommandPacket.NvmeCompletion = &Completion; 743 744 Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD; 745 CommandPacket.TransferBuffer = Private->SqBufferPciAddr[1]; 746 CommandPacket.TransferLength = EFI_PAGE_SIZE; 747 CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT; 748 CommandPacket.QueueType = NVME_ADMIN_QUEUE; 749 750 CrIoSq.Qid = NVME_IO_QUEUE; 751 CrIoSq.Qsize = NVME_CSQ_SIZE; 752 CrIoSq.Pc = 1; 753 CrIoSq.Cqid = NVME_IO_QUEUE; 754 CrIoSq.Qprio = 0; 755 CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ)); 756 CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID; 757 758 Status = Private->Passthru.PassThru ( 759 &Private->Passthru, 760 0, 761 &CommandPacket, 762 NULL 763 ); 764 765 return Status; 766 } 767 768 /** 769 Initialize the Nvm Express controller. 770 771 @param[in] Private The pointer to the NVME_CONTROLLER_PRIVATE_DATA data structure. 772 773 @retval EFI_SUCCESS The NVM Express Controller is initialized successfully. 774 @retval Others A device error occurred while initializing the controller. 775 776 **/ 777 EFI_STATUS 778 NvmeControllerInit ( 779 IN NVME_CONTROLLER_PRIVATE_DATA *Private 780 ) 781 { 782 EFI_STATUS Status; 783 EFI_PCI_IO_PROTOCOL *PciIo; 784 UINT64 Supports; 785 NVME_AQA Aqa; 786 NVME_ASQ Asq; 787 NVME_ACQ Acq; 788 789 // 790 // Save original PCI attributes and enable this controller. 791 // 792 PciIo = Private->PciIo; 793 Status = PciIo->Attributes ( 794 PciIo, 795 EfiPciIoAttributeOperationGet, 796 0, 797 &Private->PciAttributes 798 ); 799 800 if (EFI_ERROR (Status)) { 801 return Status; 802 } 803 804 Status = PciIo->Attributes ( 805 PciIo, 806 EfiPciIoAttributeOperationSupported, 807 0, 808 &Supports 809 ); 810 811 if (!EFI_ERROR (Status)) { 812 Supports &= (UINT64)EFI_PCI_DEVICE_ENABLE; 813 Status = PciIo->Attributes ( 814 PciIo, 815 EfiPciIoAttributeOperationEnable, 816 Supports, 817 NULL 818 ); 819 } 820 821 if (EFI_ERROR (Status)) { 822 DEBUG ((EFI_D_INFO, "NvmeControllerInit: failed to enable controller\n")); 823 return Status; 824 } 825 826 // 827 // Read the Controller Capabilities register and verify that the NVM command set is supported 828 // 829 Status = ReadNvmeControllerCapabilities (Private, &Private->Cap); 830 if (EFI_ERROR (Status)) { 831 return Status; 832 } 833 834 if (Private->Cap.Css != 0x01) { 835 DEBUG ((EFI_D_INFO, "NvmeControllerInit: the controller doesn't support NVMe command set\n")); 836 return EFI_UNSUPPORTED; 837 } 838 839 // 840 // Currently the driver only supports 4k page size. 841 // 842 ASSERT ((Private->Cap.Mpsmin + 12) <= EFI_PAGE_SHIFT); 843 844 Private->Cid[0] = 0; 845 Private->Cid[1] = 0; 846 847 Status = NvmeDisableController (Private); 848 849 if (EFI_ERROR(Status)) { 850 return Status; 851 } 852 853 // 854 // set number of entries admin submission & completion queues. 855 // 856 Aqa.Asqs = NVME_ASQ_SIZE; 857 Aqa.Rsvd1 = 0; 858 Aqa.Acqs = NVME_ACQ_SIZE; 859 Aqa.Rsvd2 = 0; 860 861 // 862 // Address of admin submission queue. 863 // 864 Asq = (UINT64)(UINTN)(Private->BufferPciAddr) & ~0xFFF; 865 866 // 867 // Address of admin completion queue. 868 // 869 Acq = (UINT64)(UINTN)(Private->BufferPciAddr + EFI_PAGE_SIZE) & ~0xFFF; 870 871 // 872 // Address of I/O submission & completion queue. 873 // 874 Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer); 875 Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr); 876 Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE); 877 Private->CqBufferPciAddr[0] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 1 * EFI_PAGE_SIZE); 878 Private->SqBuffer[1] = (NVME_SQ *)(UINTN)(Private->Buffer + 2 * EFI_PAGE_SIZE); 879 Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE); 880 Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE); 881 Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE); 882 883 DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer)); 884 DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs)); 885 DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs)); 886 DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0])); 887 DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0])); 888 DEBUG ((EFI_D_INFO, "I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1])); 889 DEBUG ((EFI_D_INFO, "I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1])); 890 891 // 892 // Program admin queue attributes. 893 // 894 Status = WriteNvmeAdminQueueAttributes (Private, &Aqa); 895 896 if (EFI_ERROR(Status)) { 897 return Status; 898 } 899 900 // 901 // Program admin submission queue address. 902 // 903 Status = WriteNvmeAdminSubmissionQueueBaseAddress (Private, &Asq); 904 905 if (EFI_ERROR(Status)) { 906 return Status; 907 } 908 909 // 910 // Program admin completion queue address. 911 // 912 Status = WriteNvmeAdminCompletionQueueBaseAddress (Private, &Acq); 913 914 if (EFI_ERROR(Status)) { 915 return Status; 916 } 917 918 Status = NvmeEnableController (Private); 919 if (EFI_ERROR(Status)) { 920 return Status; 921 } 922 923 // 924 // Allocate buffer for Identify Controller data 925 // 926 Private->ControllerData = (NVME_ADMIN_CONTROLLER_DATA *)AllocateZeroPool (sizeof(NVME_ADMIN_CONTROLLER_DATA)); 927 928 if (Private->ControllerData == NULL) { 929 return EFI_OUT_OF_RESOURCES; 930 } 931 932 // 933 // Get current Identify Controller Data 934 // 935 Status = NvmeIdentifyController (Private, Private->ControllerData); 936 937 if (EFI_ERROR(Status)) { 938 FreePool(Private->ControllerData); 939 Private->ControllerData = NULL; 940 return EFI_NOT_FOUND; 941 } 942 943 // 944 // Dump NvmExpress Identify Controller Data 945 // 946 Private->ControllerData->Sn[19] = 0; 947 Private->ControllerData->Mn[39] = 0; 948 DEBUG ((EFI_D_INFO, " == NVME IDENTIFY CONTROLLER DATA ==\n")); 949 DEBUG ((EFI_D_INFO, " PCI VID : 0x%x\n", Private->ControllerData->Vid)); 950 DEBUG ((EFI_D_INFO, " PCI SSVID : 0x%x\n", Private->ControllerData->Ssvid)); 951 DEBUG ((EFI_D_INFO, " SN : %a\n", (CHAR8 *)(Private->ControllerData->Sn))); 952 DEBUG ((EFI_D_INFO, " MN : %a\n", (CHAR8 *)(Private->ControllerData->Mn))); 953 DEBUG ((EFI_D_INFO, " FR : 0x%x\n", *((UINT64*)Private->ControllerData->Fr))); 954 DEBUG ((EFI_D_INFO, " RAB : 0x%x\n", Private->ControllerData->Rab)); 955 DEBUG ((EFI_D_INFO, " IEEE : 0x%x\n", *(UINT32*)Private->ControllerData->Ieee_oui)); 956 DEBUG ((EFI_D_INFO, " AERL : 0x%x\n", Private->ControllerData->Aerl)); 957 DEBUG ((EFI_D_INFO, " SQES : 0x%x\n", Private->ControllerData->Sqes)); 958 DEBUG ((EFI_D_INFO, " CQES : 0x%x\n", Private->ControllerData->Cqes)); 959 DEBUG ((EFI_D_INFO, " NN : 0x%x\n", Private->ControllerData->Nn)); 960 961 // 962 // Create one I/O completion queue. 963 // 964 Status = NvmeCreateIoCompletionQueue (Private); 965 if (EFI_ERROR(Status)) { 966 return Status; 967 } 968 969 // 970 // Create one I/O Submission queue. 971 // 972 Status = NvmeCreateIoSubmissionQueue (Private); 973 if (EFI_ERROR(Status)) { 974 return Status; 975 } 976 977 return Status; 978 } 979 980
