1 /* SPDX-License-Identifier: GPL-2.0+ */ 2 /* 3 * Copyright (C) 2017 NXP Semiconductors 4 * Copyright (C) 2017 Bin Meng <bmeng.cn (at) gmail.com> 5 */ 6 7 #ifndef __DRIVER_NVME_H__ 8 #define __DRIVER_NVME_H__ 9 10 #include <asm/io.h> 11 12 struct nvme_id_power_state { 13 __le16 max_power; /* centiwatts */ 14 __u8 rsvd2; 15 __u8 flags; 16 __le32 entry_lat; /* microseconds */ 17 __le32 exit_lat; /* microseconds */ 18 __u8 read_tput; 19 __u8 read_lat; 20 __u8 write_tput; 21 __u8 write_lat; 22 __le16 idle_power; 23 __u8 idle_scale; 24 __u8 rsvd19; 25 __le16 active_power; 26 __u8 active_work_scale; 27 __u8 rsvd23[9]; 28 }; 29 30 enum { 31 NVME_PS_FLAGS_MAX_POWER_SCALE = 1 << 0, 32 NVME_PS_FLAGS_NON_OP_STATE = 1 << 1, 33 }; 34 35 struct nvme_id_ctrl { 36 __le16 vid; 37 __le16 ssvid; 38 char sn[20]; 39 char mn[40]; 40 char fr[8]; 41 __u8 rab; 42 __u8 ieee[3]; 43 __u8 mic; 44 __u8 mdts; 45 __u16 cntlid; 46 __u32 ver; 47 __u8 rsvd84[172]; 48 __le16 oacs; 49 __u8 acl; 50 __u8 aerl; 51 __u8 frmw; 52 __u8 lpa; 53 __u8 elpe; 54 __u8 npss; 55 __u8 avscc; 56 __u8 apsta; 57 __le16 wctemp; 58 __le16 cctemp; 59 __u8 rsvd270[242]; 60 __u8 sqes; 61 __u8 cqes; 62 __u8 rsvd514[2]; 63 __le32 nn; 64 __le16 oncs; 65 __le16 fuses; 66 __u8 fna; 67 __u8 vwc; 68 __le16 awun; 69 __le16 awupf; 70 __u8 nvscc; 71 __u8 rsvd531; 72 __le16 acwu; 73 __u8 rsvd534[2]; 74 __le32 sgls; 75 __u8 rsvd540[1508]; 76 struct nvme_id_power_state psd[32]; 77 __u8 vs[1024]; 78 }; 79 80 enum { 81 NVME_CTRL_ONCS_COMPARE = 1 << 0, 82 NVME_CTRL_ONCS_WRITE_UNCORRECTABLE = 1 << 1, 83 NVME_CTRL_ONCS_DSM = 1 << 2, 84 NVME_CTRL_VWC_PRESENT = 1 << 0, 85 }; 86 87 struct nvme_lbaf { 88 __le16 ms; 89 __u8 ds; 90 __u8 rp; 91 }; 92 93 struct nvme_id_ns { 94 __le64 nsze; 95 __le64 ncap; 96 __le64 nuse; 97 __u8 nsfeat; 98 __u8 nlbaf; 99 __u8 flbas; 100 __u8 mc; 101 __u8 dpc; 102 __u8 dps; 103 __u8 nmic; 104 __u8 rescap; 105 __u8 fpi; 106 __u8 rsvd33; 107 __le16 nawun; 108 __le16 nawupf; 109 __le16 nacwu; 110 __le16 nabsn; 111 __le16 nabo; 112 __le16 nabspf; 113 __u16 rsvd46; 114 __le64 nvmcap[2]; 115 __u8 rsvd64[40]; 116 __u8 nguid[16]; 117 __u8 eui64[8]; 118 struct nvme_lbaf lbaf[16]; 119 __u8 rsvd192[192]; 120 __u8 vs[3712]; 121 }; 122 123 enum { 124 NVME_NS_FEAT_THIN = 1 << 0, 125 NVME_NS_FLBAS_LBA_MASK = 0xf, 126 NVME_NS_FLBAS_META_EXT = 0x10, 127 NVME_LBAF_RP_BEST = 0, 128 NVME_LBAF_RP_BETTER = 1, 129 NVME_LBAF_RP_GOOD = 2, 130 NVME_LBAF_RP_DEGRADED = 3, 131 NVME_NS_DPC_PI_LAST = 1 << 4, 132 NVME_NS_DPC_PI_FIRST = 1 << 3, 133 NVME_NS_DPC_PI_TYPE3 = 1 << 2, 134 NVME_NS_DPC_PI_TYPE2 = 1 << 1, 135 NVME_NS_DPC_PI_TYPE1 = 1 << 0, 136 NVME_NS_DPS_PI_FIRST = 1 << 3, 137 NVME_NS_DPS_PI_MASK = 0x7, 138 NVME_NS_DPS_PI_TYPE1 = 1, 139 NVME_NS_DPS_PI_TYPE2 = 2, 140 NVME_NS_DPS_PI_TYPE3 = 3, 141 }; 142 143 struct nvme_smart_log { 144 __u8 critical_warning; 145 __u8 temperature[2]; 146 __u8 avail_spare; 147 __u8 spare_thresh; 148 __u8 percent_used; 149 __u8 rsvd6[26]; 150 __u8 data_units_read[16]; 151 __u8 data_units_written[16]; 152 __u8 host_reads[16]; 153 __u8 host_writes[16]; 154 __u8 ctrl_busy_time[16]; 155 __u8 power_cycles[16]; 156 __u8 power_on_hours[16]; 157 __u8 unsafe_shutdowns[16]; 158 __u8 media_errors[16]; 159 __u8 num_err_log_entries[16]; 160 __le32 warning_temp_time; 161 __le32 critical_comp_time; 162 __le16 temp_sensor[8]; 163 __u8 rsvd216[296]; 164 }; 165 166 enum { 167 NVME_SMART_CRIT_SPARE = 1 << 0, 168 NVME_SMART_CRIT_TEMPERATURE = 1 << 1, 169 NVME_SMART_CRIT_RELIABILITY = 1 << 2, 170 NVME_SMART_CRIT_MEDIA = 1 << 3, 171 NVME_SMART_CRIT_VOLATILE_MEMORY = 1 << 4, 172 }; 173 174 struct nvme_lba_range_type { 175 __u8 type; 176 __u8 attributes; 177 __u8 rsvd2[14]; 178 __u64 slba; 179 __u64 nlb; 180 __u8 guid[16]; 181 __u8 rsvd48[16]; 182 }; 183 184 enum { 185 NVME_LBART_TYPE_FS = 0x01, 186 NVME_LBART_TYPE_RAID = 0x02, 187 NVME_LBART_TYPE_CACHE = 0x03, 188 NVME_LBART_TYPE_SWAP = 0x04, 189 190 NVME_LBART_ATTRIB_TEMP = 1 << 0, 191 NVME_LBART_ATTRIB_HIDE = 1 << 1, 192 }; 193 194 struct nvme_reservation_status { 195 __le32 gen; 196 __u8 rtype; 197 __u8 regctl[2]; 198 __u8 resv5[2]; 199 __u8 ptpls; 200 __u8 resv10[13]; 201 struct { 202 __le16 cntlid; 203 __u8 rcsts; 204 __u8 resv3[5]; 205 __le64 hostid; 206 __le64 rkey; 207 } regctl_ds[]; 208 }; 209 210 /* I/O commands */ 211 212 enum nvme_opcode { 213 nvme_cmd_flush = 0x00, 214 nvme_cmd_write = 0x01, 215 nvme_cmd_read = 0x02, 216 nvme_cmd_write_uncor = 0x04, 217 nvme_cmd_compare = 0x05, 218 nvme_cmd_write_zeroes = 0x08, 219 nvme_cmd_dsm = 0x09, 220 nvme_cmd_resv_register = 0x0d, 221 nvme_cmd_resv_report = 0x0e, 222 nvme_cmd_resv_acquire = 0x11, 223 nvme_cmd_resv_release = 0x15, 224 }; 225 226 struct nvme_common_command { 227 __u8 opcode; 228 __u8 flags; 229 __u16 command_id; 230 __le32 nsid; 231 __le32 cdw2[2]; 232 __le64 metadata; 233 __le64 prp1; 234 __le64 prp2; 235 __le32 cdw10[6]; 236 }; 237 238 struct nvme_rw_command { 239 __u8 opcode; 240 __u8 flags; 241 __u16 command_id; 242 __le32 nsid; 243 __u64 rsvd2; 244 __le64 metadata; 245 __le64 prp1; 246 __le64 prp2; 247 __le64 slba; 248 __le16 length; 249 __le16 control; 250 __le32 dsmgmt; 251 __le32 reftag; 252 __le16 apptag; 253 __le16 appmask; 254 }; 255 256 enum { 257 NVME_RW_LR = 1 << 15, 258 NVME_RW_FUA = 1 << 14, 259 NVME_RW_DSM_FREQ_UNSPEC = 0, 260 NVME_RW_DSM_FREQ_TYPICAL = 1, 261 NVME_RW_DSM_FREQ_RARE = 2, 262 NVME_RW_DSM_FREQ_READS = 3, 263 NVME_RW_DSM_FREQ_WRITES = 4, 264 NVME_RW_DSM_FREQ_RW = 5, 265 NVME_RW_DSM_FREQ_ONCE = 6, 266 NVME_RW_DSM_FREQ_PREFETCH = 7, 267 NVME_RW_DSM_FREQ_TEMP = 8, 268 NVME_RW_DSM_LATENCY_NONE = 0 << 4, 269 NVME_RW_DSM_LATENCY_IDLE = 1 << 4, 270 NVME_RW_DSM_LATENCY_NORM = 2 << 4, 271 NVME_RW_DSM_LATENCY_LOW = 3 << 4, 272 NVME_RW_DSM_SEQ_REQ = 1 << 6, 273 NVME_RW_DSM_COMPRESSED = 1 << 7, 274 NVME_RW_PRINFO_PRCHK_REF = 1 << 10, 275 NVME_RW_PRINFO_PRCHK_APP = 1 << 11, 276 NVME_RW_PRINFO_PRCHK_GUARD = 1 << 12, 277 NVME_RW_PRINFO_PRACT = 1 << 13, 278 }; 279 280 struct nvme_dsm_cmd { 281 __u8 opcode; 282 __u8 flags; 283 __u16 command_id; 284 __le32 nsid; 285 __u64 rsvd2[2]; 286 __le64 prp1; 287 __le64 prp2; 288 __le32 nr; 289 __le32 attributes; 290 __u32 rsvd12[4]; 291 }; 292 293 enum { 294 NVME_DSMGMT_IDR = 1 << 0, 295 NVME_DSMGMT_IDW = 1 << 1, 296 NVME_DSMGMT_AD = 1 << 2, 297 }; 298 299 struct nvme_dsm_range { 300 __le32 cattr; 301 __le32 nlb; 302 __le64 slba; 303 }; 304 305 /* Admin commands */ 306 307 enum nvme_admin_opcode { 308 nvme_admin_delete_sq = 0x00, 309 nvme_admin_create_sq = 0x01, 310 nvme_admin_get_log_page = 0x02, 311 nvme_admin_delete_cq = 0x04, 312 nvme_admin_create_cq = 0x05, 313 nvme_admin_identify = 0x06, 314 nvme_admin_abort_cmd = 0x08, 315 nvme_admin_set_features = 0x09, 316 nvme_admin_get_features = 0x0a, 317 nvme_admin_async_event = 0x0c, 318 nvme_admin_activate_fw = 0x10, 319 nvme_admin_download_fw = 0x11, 320 nvme_admin_format_nvm = 0x80, 321 nvme_admin_security_send = 0x81, 322 nvme_admin_security_recv = 0x82, 323 }; 324 325 enum { 326 NVME_QUEUE_PHYS_CONTIG = (1 << 0), 327 NVME_CQ_IRQ_ENABLED = (1 << 1), 328 NVME_SQ_PRIO_URGENT = (0 << 1), 329 NVME_SQ_PRIO_HIGH = (1 << 1), 330 NVME_SQ_PRIO_MEDIUM = (2 << 1), 331 NVME_SQ_PRIO_LOW = (3 << 1), 332 NVME_FEAT_ARBITRATION = 0x01, 333 NVME_FEAT_POWER_MGMT = 0x02, 334 NVME_FEAT_LBA_RANGE = 0x03, 335 NVME_FEAT_TEMP_THRESH = 0x04, 336 NVME_FEAT_ERR_RECOVERY = 0x05, 337 NVME_FEAT_VOLATILE_WC = 0x06, 338 NVME_FEAT_NUM_QUEUES = 0x07, 339 NVME_FEAT_IRQ_COALESCE = 0x08, 340 NVME_FEAT_IRQ_CONFIG = 0x09, 341 NVME_FEAT_WRITE_ATOMIC = 0x0a, 342 NVME_FEAT_ASYNC_EVENT = 0x0b, 343 NVME_FEAT_AUTO_PST = 0x0c, 344 NVME_FEAT_SW_PROGRESS = 0x80, 345 NVME_FEAT_HOST_ID = 0x81, 346 NVME_FEAT_RESV_MASK = 0x82, 347 NVME_FEAT_RESV_PERSIST = 0x83, 348 NVME_LOG_ERROR = 0x01, 349 NVME_LOG_SMART = 0x02, 350 NVME_LOG_FW_SLOT = 0x03, 351 NVME_LOG_RESERVATION = 0x80, 352 NVME_FWACT_REPL = (0 << 3), 353 NVME_FWACT_REPL_ACTV = (1 << 3), 354 NVME_FWACT_ACTV = (2 << 3), 355 }; 356 357 struct nvme_identify { 358 __u8 opcode; 359 __u8 flags; 360 __u16 command_id; 361 __le32 nsid; 362 __u64 rsvd2[2]; 363 __le64 prp1; 364 __le64 prp2; 365 __le32 cns; 366 __u32 rsvd11[5]; 367 }; 368 369 struct nvme_features { 370 __u8 opcode; 371 __u8 flags; 372 __u16 command_id; 373 __le32 nsid; 374 __u64 rsvd2[2]; 375 __le64 prp1; 376 __le64 prp2; 377 __le32 fid; 378 __le32 dword11; 379 __u32 rsvd12[4]; 380 }; 381 382 struct nvme_create_cq { 383 __u8 opcode; 384 __u8 flags; 385 __u16 command_id; 386 __u32 rsvd1[5]; 387 __le64 prp1; 388 __u64 rsvd8; 389 __le16 cqid; 390 __le16 qsize; 391 __le16 cq_flags; 392 __le16 irq_vector; 393 __u32 rsvd12[4]; 394 }; 395 396 struct nvme_create_sq { 397 __u8 opcode; 398 __u8 flags; 399 __u16 command_id; 400 __u32 rsvd1[5]; 401 __le64 prp1; 402 __u64 rsvd8; 403 __le16 sqid; 404 __le16 qsize; 405 __le16 sq_flags; 406 __le16 cqid; 407 __u32 rsvd12[4]; 408 }; 409 410 struct nvme_delete_queue { 411 __u8 opcode; 412 __u8 flags; 413 __u16 command_id; 414 __u32 rsvd1[9]; 415 __le16 qid; 416 __u16 rsvd10; 417 __u32 rsvd11[5]; 418 }; 419 420 struct nvme_abort_cmd { 421 __u8 opcode; 422 __u8 flags; 423 __u16 command_id; 424 __u32 rsvd1[9]; 425 __le16 sqid; 426 __u16 cid; 427 __u32 rsvd11[5]; 428 }; 429 430 struct nvme_download_firmware { 431 __u8 opcode; 432 __u8 flags; 433 __u16 command_id; 434 __u32 rsvd1[5]; 435 __le64 prp1; 436 __le64 prp2; 437 __le32 numd; 438 __le32 offset; 439 __u32 rsvd12[4]; 440 }; 441 442 struct nvme_format_cmd { 443 __u8 opcode; 444 __u8 flags; 445 __u16 command_id; 446 __le32 nsid; 447 __u64 rsvd2[4]; 448 __le32 cdw10; 449 __u32 rsvd11[5]; 450 }; 451 452 struct nvme_command { 453 union { 454 struct nvme_common_command common; 455 struct nvme_rw_command rw; 456 struct nvme_identify identify; 457 struct nvme_features features; 458 struct nvme_create_cq create_cq; 459 struct nvme_create_sq create_sq; 460 struct nvme_delete_queue delete_queue; 461 struct nvme_download_firmware dlfw; 462 struct nvme_format_cmd format; 463 struct nvme_dsm_cmd dsm; 464 struct nvme_abort_cmd abort; 465 }; 466 }; 467 468 enum { 469 NVME_SC_SUCCESS = 0x0, 470 NVME_SC_INVALID_OPCODE = 0x1, 471 NVME_SC_INVALID_FIELD = 0x2, 472 NVME_SC_CMDID_CONFLICT = 0x3, 473 NVME_SC_DATA_XFER_ERROR = 0x4, 474 NVME_SC_POWER_LOSS = 0x5, 475 NVME_SC_INTERNAL = 0x6, 476 NVME_SC_ABORT_REQ = 0x7, 477 NVME_SC_ABORT_QUEUE = 0x8, 478 NVME_SC_FUSED_FAIL = 0x9, 479 NVME_SC_FUSED_MISSING = 0xa, 480 NVME_SC_INVALID_NS = 0xb, 481 NVME_SC_CMD_SEQ_ERROR = 0xc, 482 NVME_SC_SGL_INVALID_LAST = 0xd, 483 NVME_SC_SGL_INVALID_COUNT = 0xe, 484 NVME_SC_SGL_INVALID_DATA = 0xf, 485 NVME_SC_SGL_INVALID_METADATA = 0x10, 486 NVME_SC_SGL_INVALID_TYPE = 0x11, 487 NVME_SC_LBA_RANGE = 0x80, 488 NVME_SC_CAP_EXCEEDED = 0x81, 489 NVME_SC_NS_NOT_READY = 0x82, 490 NVME_SC_RESERVATION_CONFLICT = 0x83, 491 NVME_SC_CQ_INVALID = 0x100, 492 NVME_SC_QID_INVALID = 0x101, 493 NVME_SC_QUEUE_SIZE = 0x102, 494 NVME_SC_ABORT_LIMIT = 0x103, 495 NVME_SC_ABORT_MISSING = 0x104, 496 NVME_SC_ASYNC_LIMIT = 0x105, 497 NVME_SC_FIRMWARE_SLOT = 0x106, 498 NVME_SC_FIRMWARE_IMAGE = 0x107, 499 NVME_SC_INVALID_VECTOR = 0x108, 500 NVME_SC_INVALID_LOG_PAGE = 0x109, 501 NVME_SC_INVALID_FORMAT = 0x10a, 502 NVME_SC_FIRMWARE_NEEDS_RESET = 0x10b, 503 NVME_SC_INVALID_QUEUE = 0x10c, 504 NVME_SC_FEATURE_NOT_SAVEABLE = 0x10d, 505 NVME_SC_FEATURE_NOT_CHANGEABLE = 0x10e, 506 NVME_SC_FEATURE_NOT_PER_NS = 0x10f, 507 NVME_SC_FW_NEEDS_RESET_SUBSYS = 0x110, 508 NVME_SC_BAD_ATTRIBUTES = 0x180, 509 NVME_SC_INVALID_PI = 0x181, 510 NVME_SC_READ_ONLY = 0x182, 511 NVME_SC_WRITE_FAULT = 0x280, 512 NVME_SC_READ_ERROR = 0x281, 513 NVME_SC_GUARD_CHECK = 0x282, 514 NVME_SC_APPTAG_CHECK = 0x283, 515 NVME_SC_REFTAG_CHECK = 0x284, 516 NVME_SC_COMPARE_FAILED = 0x285, 517 NVME_SC_ACCESS_DENIED = 0x286, 518 NVME_SC_DNR = 0x4000, 519 }; 520 521 struct nvme_completion { 522 __le32 result; /* Used by admin commands to return data */ 523 __u32 rsvd; 524 __le16 sq_head; /* how much of this queue may be reclaimed */ 525 __le16 sq_id; /* submission queue that generated this entry */ 526 __u16 command_id; /* of the command which completed */ 527 __le16 status; /* did the command fail, and if so, why? */ 528 }; 529 530 /* 531 * Registers should always be accessed with double word or quad word 532 * accesses. Registers with 64-bit address pointers should be written 533 * to with dword accesses by writing the low dword first (ptr[0]), 534 * then the high dword (ptr[1]) second. 535 */ 536 static inline u64 nvme_readq(__le64 volatile *regs) 537 { 538 #if BITS_PER_LONG == 64 539 return readq(regs); 540 #else 541 __u32 *ptr = (__u32 *)regs; 542 u64 val_lo = readl(ptr); 543 u64 val_hi = readl(ptr + 1); 544 545 return val_lo + (val_hi << 32); 546 #endif 547 } 548 549 static inline void nvme_writeq(const u64 val, __le64 volatile *regs) 550 { 551 #if BITS_PER_LONG == 64 552 writeq(val, regs); 553 #else 554 __u32 *ptr = (__u32 *)regs; 555 u32 val_lo = lower_32_bits(val); 556 u32 val_hi = upper_32_bits(val); 557 writel(val_lo, ptr); 558 writel(val_hi, ptr + 1); 559 #endif 560 } 561 562 struct nvme_bar { 563 __u64 cap; /* Controller Capabilities */ 564 __u32 vs; /* Version */ 565 __u32 intms; /* Interrupt Mask Set */ 566 __u32 intmc; /* Interrupt Mask Clear */ 567 __u32 cc; /* Controller Configuration */ 568 __u32 rsvd1; /* Reserved */ 569 __u32 csts; /* Controller Status */ 570 __u32 rsvd2; /* Reserved */ 571 __u32 aqa; /* Admin Queue Attributes */ 572 __u64 asq; /* Admin SQ Base Address */ 573 __u64 acq; /* Admin CQ Base Address */ 574 }; 575 576 #define NVME_CAP_MQES(cap) ((cap) & 0xffff) 577 #define NVME_CAP_TIMEOUT(cap) (((cap) >> 24) & 0xff) 578 #define NVME_CAP_STRIDE(cap) (((cap) >> 32) & 0xf) 579 #define NVME_CAP_MPSMIN(cap) (((cap) >> 48) & 0xf) 580 #define NVME_CAP_MPSMAX(cap) (((cap) >> 52) & 0xf) 581 582 #define NVME_VS(major, minor) (((major) << 16) | ((minor) << 8)) 583 584 enum { 585 NVME_CC_ENABLE = 1 << 0, 586 NVME_CC_CSS_NVM = 0 << 4, 587 NVME_CC_MPS_SHIFT = 7, 588 NVME_CC_ARB_RR = 0 << 11, 589 NVME_CC_ARB_WRRU = 1 << 11, 590 NVME_CC_ARB_VS = 7 << 11, 591 NVME_CC_SHN_NONE = 0 << 14, 592 NVME_CC_SHN_NORMAL = 1 << 14, 593 NVME_CC_SHN_ABRUPT = 2 << 14, 594 NVME_CC_SHN_MASK = 3 << 14, 595 NVME_CC_IOSQES = 6 << 16, 596 NVME_CC_IOCQES = 4 << 20, 597 NVME_CSTS_RDY = 1 << 0, 598 NVME_CSTS_CFS = 1 << 1, 599 NVME_CSTS_SHST_NORMAL = 0 << 2, 600 NVME_CSTS_SHST_OCCUR = 1 << 2, 601 NVME_CSTS_SHST_CMPLT = 2 << 2, 602 NVME_CSTS_SHST_MASK = 3 << 2, 603 }; 604 605 /* Represents an NVM Express device. Each nvme_dev is a PCI function. */ 606 struct nvme_dev { 607 struct list_head node; 608 struct nvme_queue **queues; 609 u32 __iomem *dbs; 610 int instance; 611 unsigned queue_count; 612 unsigned online_queues; 613 unsigned max_qid; 614 int q_depth; 615 u32 db_stride; 616 u32 ctrl_config; 617 struct nvme_bar __iomem *bar; 618 struct list_head namespaces; 619 char serial[20]; 620 char model[40]; 621 char firmware_rev[8]; 622 u32 max_transfer_shift; 623 u64 cap; 624 u32 stripe_size; 625 u32 page_size; 626 u8 vwc; 627 u64 *prp_pool; 628 u32 prp_entry_num; 629 u32 nn; 630 }; 631 632 /* 633 * An NVM Express namespace is equivalent to a SCSI LUN. 634 * Each namespace is operated as an independent "device". 635 */ 636 struct nvme_ns { 637 struct list_head list; 638 struct nvme_dev *dev; 639 unsigned ns_id; 640 int devnum; 641 int lba_shift; 642 u8 flbas; 643 u64 mode_select_num_blocks; 644 u32 mode_select_block_len; 645 }; 646 647 #endif /* __DRIVER_NVME_H__ */ 648
