1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) Marvell International Ltd. and its affiliates 4 */ 5 6 #include <common.h> 7 #include <i2c.h> 8 #include <spl.h> 9 #include <asm/io.h> 10 #include <asm/arch/cpu.h> 11 #include <asm/arch/soc.h> 12 13 #include "ddr3_hw_training.h" 14 15 /* 16 * Debug 17 */ 18 #define DEBUG_PBS_FULL_C(s, d, l) \ 19 DEBUG_PBS_FULL_S(s); DEBUG_PBS_FULL_D(d, l); DEBUG_PBS_FULL_S("\n") 20 #define DEBUG_PBS_C(s, d, l) \ 21 DEBUG_PBS_S(s); DEBUG_PBS_D(d, l); DEBUG_PBS_S("\n") 22 23 #ifdef MV_DEBUG_PBS 24 #define DEBUG_PBS_S(s) puts(s) 25 #define DEBUG_PBS_D(d, l) printf("%x", d) 26 #else 27 #define DEBUG_PBS_S(s) 28 #define DEBUG_PBS_D(d, l) 29 #endif 30 31 #ifdef MV_DEBUG_FULL_PBS 32 #define DEBUG_PBS_FULL_S(s) puts(s) 33 #define DEBUG_PBS_FULL_D(d, l) printf("%x", d) 34 #else 35 #define DEBUG_PBS_FULL_S(s) 36 #define DEBUG_PBS_FULL_D(d, l) 37 #endif 38 39 #if defined(MV88F78X60) || defined(MV88F672X) 40 41 /* Temp array for skew data storage */ 42 static u32 skew_array[(MAX_PUP_NUM) * DQ_NUM] = { 0 }; 43 44 /* PBS locked dq (per pup) */ 45 extern u32 pbs_locked_dq[MAX_PUP_NUM][DQ_NUM]; 46 extern u32 pbs_locked_dm[MAX_PUP_NUM]; 47 extern u32 pbs_locked_value[MAX_PUP_NUM][DQ_NUM]; 48 49 #if defined(MV88F672X) 50 extern u32 pbs_pattern[2][LEN_16BIT_PBS_PATTERN]; 51 extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN]; 52 #else 53 extern u32 pbs_pattern_32b[2][LEN_PBS_PATTERN]; 54 extern u32 pbs_pattern_64b[2][LEN_PBS_PATTERN]; 55 #endif 56 57 extern u32 pbs_dq_mapping[PUP_NUM_64BIT + 1][DQ_NUM]; 58 59 static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info, 60 u32 cur_pup, u32 pbs_pattern_idx, u32 ecc); 61 static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup, 62 u32 pbs_pattern_idx, u32 ecc); 63 static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx, 64 u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc); 65 static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx); 66 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay); 67 68 /* 69 * Name: ddr3_pbs_tx 70 * Desc: Execute the PBS TX phase. 71 * Args: dram_info ddr3 training information struct 72 * Notes: 73 * Returns: MV_OK if success, other error code if fail. 74 */ 75 int ddr3_pbs_tx(MV_DRAM_INFO *dram_info) 76 { 77 /* Array of Deskew results */ 78 79 /* 80 * Array to hold the total sum of skew from all iterations 81 * (for average purpose) 82 */ 83 u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} }; 84 85 /* 86 * Array to hold the total average skew from both patterns 87 * (for average purpose) 88 */ 89 u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} }; 90 91 u32 pbs_rep_time = 0; /* counts number of loop in case of fail */ 92 /* bit array for unlock pups - used to repeat on the RX operation */ 93 u32 cur_pup; 94 u32 max_pup; 95 u32 pbs_retry; 96 u32 pup, dq, pups, cur_max_pup, valid_pup, reg; 97 u32 pattern_idx; 98 u32 ecc; 99 /* indicates whether we need to start the loop again */ 100 int start_over; 101 102 DEBUG_PBS_S("DDR3 - PBS TX - Starting PBS TX procedure\n"); 103 104 pups = dram_info->num_of_total_pups; 105 max_pup = dram_info->num_of_total_pups; 106 107 /* Enable SW override */ 108 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | 109 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS); 110 /* [0] = 1 - Enable SW override */ 111 /* 0x15B8 - Training SW 2 Register */ 112 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 113 DEBUG_PBS_S("DDR3 - PBS RX - SW Override Enabled\n"); 114 115 reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS; 116 reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */ 117 118 /* Running twice for 2 different patterns. each patterns - 3 times */ 119 for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) { 120 DEBUG_PBS_C("DDR3 - PBS TX - Working with pattern - ", 121 pattern_idx, 1); 122 123 /* Reset sum array */ 124 for (pup = 0; pup < pups; pup++) { 125 for (dq = 0; dq < DQ_NUM; dq++) 126 skew_sum_array[pup][dq] = 0; 127 } 128 129 /* 130 * Perform PBS several of times (3 for each pattern). 131 * At the end, we'll use the average 132 */ 133 /* If there is ECC, do each PBS again with mux change */ 134 for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) { 135 for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) { 136 137 /* 138 * This parameter stores the current PUP 139 * num - ecc mode dependent - 4-8 / 1 pups 140 */ 141 cur_max_pup = (1 - ecc) * 142 dram_info->num_of_std_pups + ecc; 143 144 if (ecc) { 145 /* Only 1 pup in this case */ 146 valid_pup = 0x1; 147 } else if (cur_max_pup > 4) { 148 /* 64 bit - 8 pups */ 149 valid_pup = 0xFF; 150 } else if (cur_max_pup == 4) { 151 /* 32 bit - 4 pups */ 152 valid_pup = 0xF; 153 } else { 154 /* 16 bit - 2 pups */ 155 valid_pup = 0x3; 156 } 157 158 /* ECC Support - Switch ECC Mux on ecc=1 */ 159 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & 160 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); 161 reg |= (dram_info->ecc_ena * ecc << 162 REG_DRAM_TRAINING_2_ECC_MUX_OFFS); 163 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 164 165 if (ecc) 166 DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Enabled\n"); 167 else 168 DEBUG_PBS_S("DDR3 - PBS Tx - ECC Mux Disabled\n"); 169 170 /* Init iteration values */ 171 /* Clear the locked DQs */ 172 for (pup = 0; pup < cur_max_pup; pup++) { 173 for (dq = 0; dq < DQ_NUM; dq++) { 174 pbs_locked_dq[ 175 pup + ecc * 176 (max_pup - 1)][dq] = 177 0; 178 } 179 } 180 181 pbs_rep_time = 0; 182 cur_pup = valid_pup; 183 start_over = 0; 184 185 /* 186 * Run loop On current Pattern and current 187 * pattern iteration (just to cover the false 188 * fail problem) 189 */ 190 do { 191 DEBUG_PBS_S("DDR3 - PBS Tx - Pbs Rep Loop is "); 192 DEBUG_PBS_D(pbs_rep_time, 1); 193 DEBUG_PBS_S(", for Retry No."); 194 DEBUG_PBS_D(pbs_retry, 1); 195 DEBUG_PBS_S("\n"); 196 197 /* Set all PBS values to MIN (0) */ 198 DEBUG_PBS_S("DDR3 - PBS Tx - Set all PBS values to MIN\n"); 199 200 for (dq = 0; dq < DQ_NUM; dq++) { 201 ddr3_write_pup_reg( 202 PUP_PBS_TX + 203 pbs_dq_mapping[pup * 204 (1 - ecc) + 205 ecc * ECC_PUP] 206 [dq], CS0, (1 - ecc) * 207 PUP_BC + ecc * ECC_PUP, 0, 208 0); 209 } 210 211 /* 212 * Shift DQ ADLL right, One step before 213 * fail 214 */ 215 DEBUG_PBS_S("DDR3 - PBS Tx - ADLL shift right one phase before fail\n"); 216 217 if (MV_OK != ddr3_tx_shift_dqs_adll_step_before_fail 218 (dram_info, cur_pup, pattern_idx, 219 ecc)) 220 return MV_DDR3_TRAINING_ERR_PBS_ADLL_SHR_1PHASE; 221 222 /* PBS For each bit */ 223 DEBUG_PBS_S("DDR3 - PBS Tx - perform PBS for each bit\n"); 224 225 /* 226 * In this stage - start_over = 0 227 */ 228 if (MV_OK != ddr3_pbs_per_bit( 229 dram_info, &start_over, 1, 230 &cur_pup, pattern_idx, ecc)) 231 return MV_DDR3_TRAINING_ERR_PBS_TX_PER_BIT; 232 233 } while ((start_over == 1) && 234 (++pbs_rep_time < COUNT_PBS_STARTOVER)); 235 236 if (pbs_rep_time == COUNT_PBS_STARTOVER && 237 start_over == 1) { 238 DEBUG_PBS_S("DDR3 - PBS Tx - FAIL - Adll reach max value\n"); 239 return MV_DDR3_TRAINING_ERR_PBS_TX_MAX_VAL; 240 } 241 242 DEBUG_PBS_FULL_C("DDR3 - PBS TX - values for iteration - ", 243 pbs_retry, 1); 244 for (pup = 0; pup < cur_max_pup; pup++) { 245 /* 246 * To minimize delay elements, inc 247 * from pbs value the min pbs val 248 */ 249 DEBUG_PBS_S("DDR3 - PBS - PUP"); 250 DEBUG_PBS_D((pup + (ecc * ECC_PUP)), 1); 251 DEBUG_PBS_S(": "); 252 253 for (dq = 0; dq < DQ_NUM; dq++) { 254 /* Set skew value for all dq */ 255 /* 256 * Bit# Deskew <- Bit# Deskew - 257 * last / first failing bit 258 * Deskew For all bits (per PUP) 259 * (minimize delay elements) 260 */ 261 DEBUG_PBS_S("DQ"); 262 DEBUG_PBS_D(dq, 1); 263 DEBUG_PBS_S("-"); 264 DEBUG_PBS_D(skew_array 265 [((pup) * DQ_NUM) + 266 dq], 2); 267 DEBUG_PBS_S(", "); 268 } 269 DEBUG_PBS_S("\n"); 270 } 271 272 /* 273 * Collect the results we got on this trial 274 * of PBS 275 */ 276 for (pup = 0; pup < cur_max_pup; pup++) { 277 for (dq = 0; dq < DQ_NUM; dq++) { 278 skew_sum_array[pup + (ecc * (max_pup - 1))] 279 [dq] += skew_array 280 [((pup) * DQ_NUM) + dq]; 281 } 282 } 283 284 /* ECC Support - Disable ECC MUX */ 285 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & 286 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); 287 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 288 } 289 } 290 291 DEBUG_PBS_C("DDR3 - PBS TX - values for current pattern - ", 292 pattern_idx, 1); 293 for (pup = 0; pup < max_pup; pup++) { 294 /* 295 * To minimize delay elements, inc from pbs value the 296 * min pbs val 297 */ 298 DEBUG_PBS_S("DDR3 - PBS - PUP"); 299 DEBUG_PBS_D(pup, 1); 300 DEBUG_PBS_S(": "); 301 302 for (dq = 0; dq < DQ_NUM; dq++) { 303 /* set skew value for all dq */ 304 /* Bit# Deskew <- Bit# Deskew - last / first failing bit Deskew For all bits (per PUP) (minimize delay elements) */ 305 DEBUG_PBS_S("DQ"); 306 DEBUG_PBS_D(dq, 1); 307 DEBUG_PBS_S("-"); 308 DEBUG_PBS_D(skew_sum_array[pup][dq] / 309 COUNT_PBS_REPEAT, 2); 310 DEBUG_PBS_S(", "); 311 } 312 DEBUG_PBS_S("\n"); 313 } 314 315 /* 316 * Calculate the average skew for current pattern for each 317 * pup and each bit 318 */ 319 DEBUG_PBS_C("DDR3 - PBS TX - Average for pattern - ", 320 pattern_idx, 1); 321 322 for (pup = 0; pup < max_pup; pup++) { 323 /* 324 * FOR ECC only :: found min and max value for current 325 * pattern skew array 326 */ 327 /* Loop for all dqs */ 328 for (dq = 0; dq < DQ_NUM; dq++) { 329 pattern_skew_array[pup][dq] += 330 (skew_sum_array[pup][dq] / 331 COUNT_PBS_REPEAT); 332 } 333 } 334 } 335 336 /* Calculate the average skew */ 337 for (pup = 0; pup < max_pup; pup++) { 338 for (dq = 0; dq < DQ_NUM; dq++) 339 skew_array[((pup) * DQ_NUM) + dq] = 340 pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN; 341 } 342 343 DEBUG_PBS_S("DDR3 - PBS TX - Average for all patterns:\n"); 344 for (pup = 0; pup < max_pup; pup++) { 345 /* 346 * To minimize delay elements, inc from pbs value the min 347 * pbs val 348 */ 349 DEBUG_PBS_S("DDR3 - PBS - PUP"); 350 DEBUG_PBS_D(pup, 1); 351 DEBUG_PBS_S(": "); 352 353 for (dq = 0; dq < DQ_NUM; dq++) { 354 /* Set skew value for all dq */ 355 /* 356 * Bit# Deskew <- Bit# Deskew - last / first 357 * failing bit Deskew For all bits (per PUP) 358 * (minimize delay elements) 359 */ 360 DEBUG_PBS_S("DQ"); 361 DEBUG_PBS_D(dq, 1); 362 DEBUG_PBS_S("-"); 363 DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2); 364 DEBUG_PBS_S(", "); 365 } 366 DEBUG_PBS_S("\n"); 367 } 368 369 /* Return ADLL to default value */ 370 for (pup = 0; pup < max_pup; pup++) { 371 if (pup == (max_pup - 1) && dram_info->ecc_ena) 372 pup = ECC_PUP; 373 ddr3_pbs_write_pup_dqs_reg(CS0, pup, INIT_WL_DELAY); 374 } 375 376 /* Set averaged PBS results */ 377 ddr3_set_pbs_results(dram_info, 1); 378 379 /* Disable SW override - Must be in a different stage */ 380 /* [0]=0 - Enable SW override */ 381 reg = reg_read(REG_DRAM_TRAINING_2_ADDR); 382 reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS); 383 /* 0x15B8 - Training SW 2 Register */ 384 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 385 386 reg = reg_read(REG_DRAM_TRAINING_1_ADDR) | 387 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS); 388 reg_write(REG_DRAM_TRAINING_1_ADDR, reg); 389 390 DEBUG_PBS_S("DDR3 - PBS Tx - PBS TX ended successfuly\n"); 391 392 return MV_OK; 393 } 394 395 /* 396 * Name: ddr3_tx_shift_dqs_adll_step_before_fail 397 * Desc: Execute the Tx shift DQ phase. 398 * Args: dram_info ddr3 training information struct 399 * cur_pup bit array of the function active pups. 400 * pbs_pattern_idx Index of PBS pattern 401 * Notes: 402 * Returns: MV_OK if success, other error code if fail. 403 */ 404 static int ddr3_tx_shift_dqs_adll_step_before_fail(MV_DRAM_INFO *dram_info, 405 u32 cur_pup, 406 u32 pbs_pattern_idx, u32 ecc) 407 { 408 u32 unlock_pup; /* bit array of unlock pups */ 409 u32 new_lockup_pup; /* bit array of compare failed pups */ 410 u32 adll_val = 4; /* INIT_WL_DELAY */ 411 u32 cur_max_pup, pup; 412 u32 dqs_dly_set[MAX_PUP_NUM] = { 0 }; 413 u32 *pattern_ptr; 414 415 /* Choose pattern */ 416 switch (dram_info->ddr_width) { 417 #if defined(MV88F672X) 418 case 16: 419 pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx]; 420 break; 421 #endif 422 case 32: 423 pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx]; 424 break; 425 #if defined(MV88F78X60) 426 case 64: 427 pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx]; 428 break; 429 #endif 430 default: 431 return MV_FAIL; 432 } 433 434 /* Set current pup number */ 435 if (cur_pup == 0x1) /* Ecc mode */ 436 cur_max_pup = 1; 437 else 438 cur_max_pup = dram_info->num_of_std_pups; 439 440 unlock_pup = cur_pup; /* '1' for each unlocked pup */ 441 442 /* Loop on all ADLL Vaules */ 443 do { 444 /* Loop until found first fail */ 445 adll_val++; 446 447 /* 448 * Increment (Move to right - ADLL) DQ TX delay 449 * (broadcast to all Data PUPs) 450 */ 451 for (pup = 0; pup < cur_max_pup; pup++) 452 ddr3_pbs_write_pup_dqs_reg(CS0, 453 pup * (1 - ecc) + 454 ECC_PUP * ecc, adll_val); 455 456 /* 457 * Write and Read, compare results (read was already verified) 458 */ 459 /* 0 - all locked */ 460 new_lockup_pup = 0; 461 462 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup, 463 &new_lockup_pup, 464 pattern_ptr, LEN_PBS_PATTERN, 465 SDRAM_PBS_TX_OFFS, 1, 0, 466 NULL, 467 0)) 468 return MV_FAIL; 469 470 unlock_pup &= ~new_lockup_pup; 471 472 DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: "); 473 DEBUG_PBS_FULL_D(unlock_pup, 2); 474 DEBUG_PBS_FULL_C(", Set ADLL value = ", adll_val, 2); 475 476 /* If any PUP failed there is '1' to mark the PUP */ 477 if (new_lockup_pup != 0) { 478 /* 479 * Decrement (Move Back to Left two steps - ADLL) 480 * DQ TX delay for current failed pups and save 481 */ 482 for (pup = 0; pup < cur_max_pup; pup++) { 483 if (((new_lockup_pup >> pup) & 0x1) && 484 dqs_dly_set[pup] == 0) 485 dqs_dly_set[pup] = adll_val - 1; 486 } 487 } 488 } while ((unlock_pup != 0) && (adll_val != ADLL_MAX)); 489 490 if (unlock_pup != 0) { 491 DEBUG_PBS_FULL_S("DDR3 - PBS Tx - Shift DQ - Adll value reached maximum\n"); 492 493 for (pup = 0; pup < cur_max_pup; pup++) { 494 if (((unlock_pup >> pup) & 0x1) && 495 dqs_dly_set[pup] == 0) 496 dqs_dly_set[pup] = adll_val - 1; 497 } 498 } 499 500 DEBUG_PBS_FULL_C("PBS TX one step before fail last pups locked Adll ", 501 adll_val - 2, 2); 502 503 /* Set the PUP DQS DLY Values */ 504 for (pup = 0; pup < cur_max_pup; pup++) 505 ddr3_pbs_write_pup_dqs_reg(CS0, pup * (1 - ecc) + ECC_PUP * ecc, 506 dqs_dly_set[pup]); 507 508 /* Found one phase before fail */ 509 return MV_OK; 510 } 511 512 /* 513 * Name: ddr3_pbs_rx 514 * Desc: Execute the PBS RX phase. 515 * Args: dram_info ddr3 training information struct 516 * Notes: 517 * Returns: MV_OK if success, other error code if fail. 518 */ 519 int ddr3_pbs_rx(MV_DRAM_INFO *dram_info) 520 { 521 /* 522 * Array to hold the total sum of skew from all iterations 523 * (for average purpose) 524 */ 525 u32 skew_sum_array[MAX_PUP_NUM][DQ_NUM] = { {0} }; 526 527 /* 528 * Array to hold the total average skew from both patterns 529 * (for average purpose) 530 */ 531 u32 pattern_skew_array[MAX_PUP_NUM][DQ_NUM] = { {0} }; 532 533 u32 pbs_rep_time = 0; /* counts number of loop in case of fail */ 534 /* bit array for unlock pups - used to repeat on the RX operation */ 535 u32 cur_pup; 536 u32 max_pup; 537 u32 pbs_retry; 538 u32 pup, dq, pups, cur_max_pup, valid_pup, reg; 539 u32 pattern_idx; 540 u32 ecc; 541 /* indicates whether we need to start the loop again */ 542 int start_over; 543 int status; 544 545 DEBUG_PBS_S("DDR3 - PBS RX - Starting PBS RX procedure\n"); 546 547 pups = dram_info->num_of_total_pups; 548 max_pup = dram_info->num_of_total_pups; 549 550 /* Enable SW override */ 551 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | 552 (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS); 553 /* [0] = 1 - Enable SW override */ 554 /* 0x15B8 - Training SW 2 Register */ 555 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 556 DEBUG_PBS_FULL_S("DDR3 - PBS RX - SW Override Enabled\n"); 557 558 reg = 1 << REG_DRAM_TRAINING_AUTO_OFFS; 559 reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */ 560 561 /* Running twice for 2 different patterns. each patterns - 3 times */ 562 for (pattern_idx = 0; pattern_idx < COUNT_PBS_PATTERN; pattern_idx++) { 563 DEBUG_PBS_FULL_C("DDR3 - PBS RX - Working with pattern - ", 564 pattern_idx, 1); 565 566 /* Reset sum array */ 567 for (pup = 0; pup < pups; pup++) { 568 for (dq = 0; dq < DQ_NUM; dq++) 569 skew_sum_array[pup][dq] = 0; 570 } 571 572 /* 573 * Perform PBS several of times (3 for each pattern). 574 * At the end, we'll use the average 575 */ 576 /* If there is ECC, do each PBS again with mux change */ 577 for (pbs_retry = 0; pbs_retry < COUNT_PBS_REPEAT; pbs_retry++) { 578 for (ecc = 0; ecc < (dram_info->ecc_ena + 1); ecc++) { 579 /* 580 * This parameter stores the current PUP 581 * num - ecc mode dependent - 4-8 / 1 pups 582 */ 583 cur_max_pup = (1 - ecc) * 584 dram_info->num_of_std_pups + ecc; 585 586 if (ecc) { 587 /* Only 1 pup in this case */ 588 valid_pup = 0x1; 589 } else if (cur_max_pup > 4) { 590 /* 64 bit - 8 pups */ 591 valid_pup = 0xFF; 592 } else if (cur_max_pup == 4) { 593 /* 32 bit - 4 pups */ 594 valid_pup = 0xF; 595 } else { 596 /* 16 bit - 2 pups */ 597 valid_pup = 0x3; 598 } 599 600 /* ECC Support - Switch ECC Mux on ecc=1 */ 601 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & 602 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); 603 reg |= (dram_info->ecc_ena * ecc << 604 REG_DRAM_TRAINING_2_ECC_MUX_OFFS); 605 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 606 607 if (ecc) 608 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Enabled\n"); 609 else 610 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - ECC Mux Disabled\n"); 611 612 /* Init iteration values */ 613 /* Clear the locked DQs */ 614 for (pup = 0; pup < cur_max_pup; pup++) { 615 for (dq = 0; dq < DQ_NUM; dq++) { 616 pbs_locked_dq[ 617 pup + ecc * (max_pup - 1)][dq] = 618 0; 619 } 620 } 621 622 pbs_rep_time = 0; 623 cur_pup = valid_pup; 624 start_over = 0; 625 626 /* 627 * Run loop On current Pattern and current 628 * pattern iteration (just to cover the false 629 * fail problem 630 */ 631 do { 632 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Pbs Rep Loop is "); 633 DEBUG_PBS_FULL_D(pbs_rep_time, 1); 634 DEBUG_PBS_FULL_S(", for Retry No."); 635 DEBUG_PBS_FULL_D(pbs_retry, 1); 636 DEBUG_PBS_FULL_S("\n"); 637 638 /* Set all PBS values to MAX (31) */ 639 for (pup = 0; pup < cur_max_pup; pup++) { 640 for (dq = 0; dq < DQ_NUM; dq++) 641 ddr3_write_pup_reg( 642 PUP_PBS_RX + 643 pbs_dq_mapping[ 644 pup * (1 - ecc) 645 + ecc * ECC_PUP] 646 [dq], CS0, 647 pup + ecc * ECC_PUP, 648 0, MAX_PBS); 649 } 650 651 /* Set all DQS PBS values to MIN (0) */ 652 for (pup = 0; pup < cur_max_pup; pup++) { 653 ddr3_write_pup_reg(PUP_PBS_RX + 654 DQ_NUM, CS0, 655 pup + 656 ecc * 657 ECC_PUP, 0, 658 0); 659 } 660 661 /* Shift DQS, To first Fail */ 662 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift RX DQS to first fail\n"); 663 664 status = ddr3_rx_shift_dqs_to_first_fail 665 (dram_info, cur_pup, 666 pattern_idx, ecc); 667 if (MV_OK != status) { 668 DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_rx_shift_dqs_to_first_fail failed.\n"); 669 DEBUG_PBS_D(status, 8); 670 DEBUG_PBS_S("\nDDR3 - PBS Rx - SKIP.\n"); 671 672 /* Reset read FIFO */ 673 reg = reg_read(REG_DRAM_TRAINING_ADDR); 674 /* Start Auto Read Leveling procedure */ 675 reg |= (1 << REG_DRAM_TRAINING_RL_OFFS); 676 /* 0x15B0 - Training Register */ 677 reg_write(REG_DRAM_TRAINING_ADDR, reg); 678 679 reg = reg_read(REG_DRAM_TRAINING_2_ADDR); 680 reg |= ((1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS) 681 + (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS)); 682 /* [0] = 1 - Enable SW override, [4] = 1 - FIFO reset */ 683 /* 0x15B8 - Training SW 2 Register */ 684 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 685 686 do { 687 reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) 688 & (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); 689 } while (reg); /* Wait for '0' */ 690 691 reg = reg_read(REG_DRAM_TRAINING_ADDR); 692 /* Clear Auto Read Leveling procedure */ 693 reg &= ~(1 << REG_DRAM_TRAINING_RL_OFFS); 694 /* 0x15B0 - Training Register */ 695 reg_write(REG_DRAM_TRAINING_ADDR, reg); 696 697 /* Set ADLL to 15 */ 698 for (pup = 0; pup < max_pup; 699 pup++) { 700 ddr3_write_pup_reg 701 (PUP_DQS_RD, CS0, 702 pup + 703 (ecc * ECC_PUP), 0, 704 15); 705 } 706 707 /* Set all PBS values to MIN (0) */ 708 for (pup = 0; pup < cur_max_pup; 709 pup++) { 710 for (dq = 0; 711 dq < DQ_NUM; dq++) 712 ddr3_write_pup_reg 713 (PUP_PBS_RX + 714 pbs_dq_mapping 715 [pup * (1 - ecc) + 716 ecc * ECC_PUP] 717 [dq], CS0, 718 pup + ecc * ECC_PUP, 719 0, MIN_PBS); 720 } 721 722 return MV_OK; 723 } 724 725 /* PBS For each bit */ 726 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - perform PBS for each bit\n"); 727 /* in this stage - start_over = 0; */ 728 if (MV_OK != ddr3_pbs_per_bit( 729 dram_info, &start_over, 730 0, &cur_pup, 731 pattern_idx, ecc)) { 732 DEBUG_PBS_S("DDR3 - PBS Rx - ddr3_pbs_per_bit failed."); 733 return MV_DDR3_TRAINING_ERR_PBS_RX_PER_BIT; 734 } 735 736 } while ((start_over == 1) && 737 (++pbs_rep_time < COUNT_PBS_STARTOVER)); 738 739 if (pbs_rep_time == COUNT_PBS_STARTOVER && 740 start_over == 1) { 741 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - FAIL - Algorithm failed doing RX PBS\n"); 742 return MV_DDR3_TRAINING_ERR_PBS_RX_MAX_VAL; 743 } 744 745 /* Return DQS ADLL to default value - 15 */ 746 /* Set all DQS PBS values to MIN (0) */ 747 for (pup = 0; pup < cur_max_pup; pup++) 748 ddr3_write_pup_reg(PUP_DQS_RD, CS0, 749 pup + ecc * ECC_PUP, 750 0, INIT_RL_DELAY); 751 752 DEBUG_PBS_FULL_C("DDR3 - PBS RX - values for iteration - ", 753 pbs_retry, 1); 754 for (pup = 0; pup < cur_max_pup; pup++) { 755 /* 756 * To minimize delay elements, inc from 757 * pbs value the min pbs val 758 */ 759 DEBUG_PBS_FULL_S("DDR3 - PBS - PUP"); 760 DEBUG_PBS_FULL_D((pup + 761 (ecc * ECC_PUP)), 1); 762 DEBUG_PBS_FULL_S(": "); 763 764 for (dq = 0; dq < DQ_NUM; dq++) { 765 /* Set skew value for all dq */ 766 /* 767 * Bit# Deskew <- Bit# Deskew - 768 * last / first failing bit 769 * Deskew For all bits (per PUP) 770 * (minimize delay elements) 771 */ 772 DEBUG_PBS_FULL_S("DQ"); 773 DEBUG_PBS_FULL_D(dq, 1); 774 DEBUG_PBS_FULL_S("-"); 775 DEBUG_PBS_FULL_D(skew_array 776 [((pup) * 777 DQ_NUM) + 778 dq], 2); 779 DEBUG_PBS_FULL_S(", "); 780 } 781 DEBUG_PBS_FULL_S("\n"); 782 } 783 784 /* 785 * Collect the results we got on this trial 786 * of PBS 787 */ 788 for (pup = 0; pup < cur_max_pup; pup++) { 789 for (dq = 0; dq < DQ_NUM; dq++) { 790 skew_sum_array 791 [pup + (ecc * (max_pup - 1))] 792 [dq] += 793 skew_array[((pup) * DQ_NUM) + dq]; 794 } 795 } 796 797 /* ECC Support - Disable ECC MUX */ 798 reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & 799 ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); 800 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 801 } 802 } 803 804 /* 805 * Calculate the average skew for current pattern for each 806 * pup and each bit 807 */ 808 DEBUG_PBS_FULL_C("DDR3 - PBS RX - Average for pattern - ", 809 pattern_idx, 1); 810 for (pup = 0; pup < max_pup; pup++) { 811 /* 812 * FOR ECC only :: found min and max value for 813 * current pattern skew array 814 */ 815 /* Loop for all dqs */ 816 for (dq = 0; dq < DQ_NUM; dq++) { 817 pattern_skew_array[pup][dq] += 818 (skew_sum_array[pup][dq] / 819 COUNT_PBS_REPEAT); 820 } 821 } 822 823 DEBUG_PBS_C("DDR3 - PBS RX - values for current pattern - ", 824 pattern_idx, 1); 825 for (pup = 0; pup < max_pup; pup++) { 826 /* 827 * To minimize delay elements, inc from pbs value the 828 * min pbs val 829 */ 830 DEBUG_PBS_S("DDR3 - PBS RX - PUP"); 831 DEBUG_PBS_D(pup, 1); 832 DEBUG_PBS_S(": "); 833 834 for (dq = 0; dq < DQ_NUM; dq++) { 835 /* Set skew value for all dq */ 836 /* 837 * Bit# Deskew <- Bit# Deskew - last / first 838 * failing bit Deskew For all bits (per PUP) 839 * (minimize delay elements) 840 */ 841 DEBUG_PBS_S("DQ"); 842 DEBUG_PBS_D(dq, 1); 843 DEBUG_PBS_S("-"); 844 DEBUG_PBS_D(skew_sum_array[pup][dq] / 845 COUNT_PBS_REPEAT, 2); 846 DEBUG_PBS_S(", "); 847 } 848 DEBUG_PBS_S("\n"); 849 } 850 } 851 852 /* Calculate the average skew */ 853 for (pup = 0; pup < max_pup; pup++) { 854 for (dq = 0; dq < DQ_NUM; dq++) 855 skew_array[((pup) * DQ_NUM) + dq] = 856 pattern_skew_array[pup][dq] / COUNT_PBS_PATTERN; 857 } 858 859 DEBUG_PBS_S("DDR3 - PBS RX - Average for all patterns:\n"); 860 for (pup = 0; pup < max_pup; pup++) { 861 /* 862 * To minimize delay elements, inc from pbs value the 863 * min pbs val 864 */ 865 DEBUG_PBS_S("DDR3 - PBS - PUP"); 866 DEBUG_PBS_D(pup, 1); 867 DEBUG_PBS_S(": "); 868 869 for (dq = 0; dq < DQ_NUM; dq++) { 870 /* Set skew value for all dq */ 871 /* 872 * Bit# Deskew <- Bit# Deskew - last / first 873 * failing bit Deskew For all bits (per PUP) 874 * (minimize delay elements) 875 */ 876 DEBUG_PBS_S("DQ"); 877 DEBUG_PBS_D(dq, 1); 878 DEBUG_PBS_S("-"); 879 DEBUG_PBS_D(skew_array[(pup * DQ_NUM) + dq], 2); 880 DEBUG_PBS_S(", "); 881 } 882 DEBUG_PBS_S("\n"); 883 } 884 885 /* Return ADLL to default value */ 886 ddr3_write_pup_reg(PUP_DQS_RD, CS0, PUP_BC, 0, INIT_RL_DELAY); 887 888 /* Set averaged PBS results */ 889 ddr3_set_pbs_results(dram_info, 0); 890 891 /* Disable SW override - Must be in a different stage */ 892 /* [0]=0 - Enable SW override */ 893 reg = reg_read(REG_DRAM_TRAINING_2_ADDR); 894 reg &= ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS); 895 /* 0x15B8 - Training SW 2 Register */ 896 reg_write(REG_DRAM_TRAINING_2_ADDR, reg); 897 898 reg = reg_read(REG_DRAM_TRAINING_1_ADDR) | 899 (1 << REG_DRAM_TRAINING_1_TRNBPOINT_OFFS); 900 reg_write(REG_DRAM_TRAINING_1_ADDR, reg); 901 902 DEBUG_PBS_FULL_S("DDR3 - PBS RX - ended successfuly\n"); 903 904 return MV_OK; 905 } 906 907 /* 908 * Name: ddr3_rx_shift_dqs_to_first_fail 909 * Desc: Execute the Rx shift DQ phase. 910 * Args: dram_info ddr3 training information struct 911 * cur_pup bit array of the function active pups. 912 * pbs_pattern_idx Index of PBS pattern 913 * Notes: 914 * Returns: MV_OK if success, other error code if fail. 915 */ 916 static int ddr3_rx_shift_dqs_to_first_fail(MV_DRAM_INFO *dram_info, u32 cur_pup, 917 u32 pbs_pattern_idx, u32 ecc) 918 { 919 u32 unlock_pup; /* bit array of unlock pups */ 920 u32 new_lockup_pup; /* bit array of compare failed pups */ 921 u32 adll_val = MAX_DELAY; 922 u32 dqs_deskew_val = 0; /* current value of DQS PBS deskew */ 923 u32 cur_max_pup, pup, pass_pup; 924 u32 *pattern_ptr; 925 926 /* Choose pattern */ 927 switch (dram_info->ddr_width) { 928 #if defined(MV88F672X) 929 case 16: 930 pattern_ptr = (u32 *)&pbs_pattern[pbs_pattern_idx]; 931 break; 932 #endif 933 case 32: 934 pattern_ptr = (u32 *)&pbs_pattern_32b[pbs_pattern_idx]; 935 break; 936 #if defined(MV88F78X60) 937 case 64: 938 pattern_ptr = (u32 *)&pbs_pattern_64b[pbs_pattern_idx]; 939 break; 940 #endif 941 default: 942 return MV_FAIL; 943 } 944 945 /* Set current pup number */ 946 if (cur_pup == 0x1) /* Ecc mode */ 947 cur_max_pup = 1; 948 else 949 cur_max_pup = dram_info->num_of_std_pups; 950 951 unlock_pup = cur_pup; /* '1' for each unlocked pup */ 952 953 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Starting...\n"); 954 955 /* Set DQS ADLL to MAX */ 956 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Set DQS ADLL to Max for all PUPs\n"); 957 for (pup = 0; pup < cur_max_pup; pup++) 958 ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP, 0, 959 MAX_DELAY); 960 961 /* Loop on all ADLL Vaules */ 962 do { 963 /* Loop until found fail for all pups */ 964 new_lockup_pup = 0; 965 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup, 966 &new_lockup_pup, 967 pattern_ptr, LEN_PBS_PATTERN, 968 SDRAM_PBS_I_OFFS + 969 pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS, 970 0, 0, NULL, 0)) { 971 DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n"); 972 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP; 973 } 974 975 if ((new_lockup_pup != 0) && (dqs_deskew_val <= 1)) { 976 /* Fail on start with first deskew value */ 977 /* Decrement DQS ADLL */ 978 --adll_val; 979 if (adll_val == ADLL_MIN) { 980 DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - fail on start with first deskew value\n"); 981 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP; 982 } 983 ddr3_write_pup_reg(PUP_DQS_RD, CS0, pup + ecc * ECC_PUP, 984 0, adll_val); 985 continue; 986 } 987 988 /* Update all new locked pups */ 989 unlock_pup &= ~new_lockup_pup; 990 991 if ((unlock_pup == 0) || (dqs_deskew_val == MAX_PBS)) { 992 if (dqs_deskew_val == MAX_PBS) { 993 /* 994 * Reach max value of dqs deskew or get fail 995 * for all pups 996 */ 997 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - DQS deskew reached maximum value\n"); 998 } 999 break; 1000 } 1001 1002 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - Inc DQS deskew for PUPs: "); 1003 DEBUG_PBS_FULL_D(unlock_pup, 2); 1004 DEBUG_PBS_FULL_C(", deskew = ", dqs_deskew_val, 2); 1005 1006 /* Increment DQS deskew elements - Only for unlocked pups */ 1007 dqs_deskew_val++; 1008 for (pup = 0; pup < cur_max_pup; pup++) { 1009 if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) { 1010 ddr3_write_pup_reg(PUP_PBS_RX + DQS_DQ_NUM, CS0, 1011 pup + ecc * ECC_PUP, 0, 1012 dqs_deskew_val); 1013 } 1014 } 1015 } while (1); 1016 1017 DEBUG_PBS_FULL_S("DDR3 - PBS RX - Shift DQS - ADLL shift one step before fail\n"); 1018 /* Continue to ADLL shift one step before fail */ 1019 unlock_pup = cur_pup; 1020 do { 1021 /* Loop until pass compare for all pups */ 1022 new_lockup_pup = 0; 1023 /* Read and compare results */ 1024 if (MV_OK != ddr3_sdram_compare(dram_info, unlock_pup, &new_lockup_pup, 1025 pattern_ptr, LEN_PBS_PATTERN, 1026 SDRAM_PBS_I_OFFS + 1027 pbs_pattern_idx * SDRAM_PBS_NEXT_OFFS, 1028 1, 0, NULL, 0)) { 1029 DEBUG_PBS_S("DDR3 - PBS Rx - Shift DQS - MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP(ddr3_sdram_compare)\n"); 1030 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_SRAM_CMP; 1031 } 1032 1033 /* 1034 * Get mask for pup which passed so their adll will be 1035 * changed to 2 steps before fails 1036 */ 1037 pass_pup = unlock_pup & ~new_lockup_pup; 1038 1039 DEBUG_PBS_FULL_S("Shift DQS by 2 steps for PUPs: "); 1040 DEBUG_PBS_FULL_D(pass_pup, 2); 1041 DEBUG_PBS_FULL_C(", Set ADLL value = ", (adll_val - 2), 2); 1042 1043 /* Only for pass pups */ 1044 for (pup = 0; pup < cur_max_pup; pup++) { 1045 if (IS_PUP_ACTIVE(pass_pup, pup) == 1) { 1046 ddr3_write_pup_reg(PUP_DQS_RD, CS0, 1047 pup + ecc * ECC_PUP, 0, 1048 (adll_val - 2)); 1049 } 1050 } 1051 1052 /* Locked pups that compare success */ 1053 unlock_pup &= new_lockup_pup; 1054 1055 if (unlock_pup == 0) { 1056 /* All pups locked */ 1057 break; 1058 } 1059 1060 /* Found error */ 1061 if (adll_val == 0) { 1062 DEBUG_PBS_FULL_S("DDR3 - PBS Rx - Shift DQS - Adll reach min value\n"); 1063 return MV_DDR3_TRAINING_ERR_PBS_SHIFT_QDS_MAX_VAL; 1064 } 1065 1066 /* 1067 * Decrement (Move Back to Left one phase - ADLL) dqs RX delay 1068 */ 1069 adll_val--; 1070 for (pup = 0; pup < cur_max_pup; pup++) { 1071 if (IS_PUP_ACTIVE(unlock_pup, pup) == 1) { 1072 ddr3_write_pup_reg(PUP_DQS_RD, CS0, 1073 pup + ecc * ECC_PUP, 0, 1074 adll_val); 1075 } 1076 } 1077 } while (1); 1078 1079 return MV_OK; 1080 } 1081 1082 /* 1083 * lock_pups() extracted from ddr3_pbs_per_bit(). This just got too 1084 * much indented making it hard to read / edit. 1085 */ 1086 static void lock_pups(u32 pup, u32 *pup_locked, u8 *unlock_pup_dq_array, 1087 u32 pbs_curr_val, u32 start_pbs, u32 ecc, int is_tx) 1088 { 1089 u32 dq; 1090 int idx; 1091 1092 /* Lock PBS value for all remaining PUPs bits */ 1093 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Lock PBS value for all remaining PUPs bits, pup "); 1094 DEBUG_PBS_FULL_D(pup, 1); 1095 DEBUG_PBS_FULL_C(" pbs value ", pbs_curr_val, 2); 1096 1097 idx = pup * (1 - ecc) + ecc * ECC_PUP; 1098 *pup_locked &= ~(1 << pup); 1099 1100 for (dq = 0; dq < DQ_NUM; dq++) { 1101 if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup) == 1) { 1102 int offs; 1103 1104 /* Lock current dq */ 1105 unlock_pup_dq_array[dq] &= ~(1 << pup); 1106 skew_array[(pup * DQ_NUM) + dq] = pbs_curr_val; 1107 1108 if (is_tx == 1) 1109 offs = PUP_PBS_TX; 1110 else 1111 offs = PUP_PBS_RX; 1112 1113 ddr3_write_pup_reg(offs + 1114 pbs_dq_mapping[idx][dq], CS0, 1115 idx, 0, start_pbs); 1116 } 1117 } 1118 } 1119 1120 /* 1121 * Name: ddr3_pbs_per_bit 1122 * Desc: Execute the Per Bit Skew phase. 1123 * Args: start_over Return whether need to start over the algorithm 1124 * is_tx Indicate whether Rx or Tx 1125 * pcur_pup bit array of the function active pups. return the 1126 * pups that need to repeat on the PBS 1127 * pbs_pattern_idx Index of PBS pattern 1128 * 1129 * Notes: Current implementation supports double activation of this function. 1130 * i.e. in order to activate this function (using start_over) more than 1131 * twice, the implementation should change. 1132 * imlementation limitation are marked using 1133 * ' CHIP-ONLY! - Implementation Limitation ' 1134 * Returns: MV_OK if success, other error code if fail. 1135 */ 1136 static int ddr3_pbs_per_bit(MV_DRAM_INFO *dram_info, int *start_over, int is_tx, 1137 u32 *pcur_pup, u32 pbs_pattern_idx, u32 ecc) 1138 { 1139 /* 1140 * Bit array to indicate if we already get fail on bit per pup & dq bit 1141 */ 1142 u8 unlock_pup_dq_array[DQ_NUM] = { 1143 *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup, *pcur_pup, 1144 *pcur_pup, *pcur_pup, *pcur_pup 1145 }; 1146 1147 u8 cmp_unlock_pup_dq_array[COUNT_PBS_COMP_RETRY_NUM][DQ_NUM]; 1148 u32 pup, dq; 1149 /* value of pbs is according to RX or TX */ 1150 u32 start_pbs, last_pbs; 1151 u32 pbs_curr_val; 1152 /* bit array that indicates all dq of the pup locked */ 1153 u32 pup_locked; 1154 u32 first_fail[MAX_PUP_NUM] = { 0 }; /* count first fail per pup */ 1155 /* indicates whether we get first fail per pup */ 1156 int first_failed[MAX_PUP_NUM] = { 0 }; 1157 /* bit array that indicates pup already get fail */ 1158 u32 sum_pup_fail; 1159 /* use to calculate diff between curr pbs to first fail pbs */ 1160 u32 calc_pbs_diff; 1161 u32 pbs_cmp_retry; 1162 u32 max_pup; 1163 1164 /* Set init values for retry array - 8 retry */ 1165 for (pbs_cmp_retry = 0; pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM; 1166 pbs_cmp_retry++) { 1167 for (dq = 0; dq < DQ_NUM; dq++) 1168 cmp_unlock_pup_dq_array[pbs_cmp_retry][dq] = *pcur_pup; 1169 } 1170 1171 memset(&skew_array, 0, MAX_PUP_NUM * DQ_NUM * sizeof(u32)); 1172 1173 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Started\n"); 1174 1175 /* The pbs value depends if rx or tx */ 1176 if (is_tx == 1) { 1177 start_pbs = MIN_PBS; 1178 last_pbs = MAX_PBS; 1179 } else { 1180 start_pbs = MAX_PBS; 1181 last_pbs = MIN_PBS; 1182 } 1183 1184 pbs_curr_val = start_pbs; 1185 pup_locked = *pcur_pup; 1186 1187 /* Set current pup number */ 1188 if (pup_locked == 0x1) /* Ecc mode */ 1189 max_pup = 1; 1190 else 1191 max_pup = dram_info->num_of_std_pups; 1192 1193 do { 1194 /* Increment/ decrement PBS for un-lock bits only */ 1195 if (is_tx == 1) 1196 pbs_curr_val++; 1197 else 1198 pbs_curr_val--; 1199 1200 /* Set Current PBS delay */ 1201 for (dq = 0; dq < DQ_NUM; dq++) { 1202 /* Check DQ bits to see if locked in all pups */ 1203 if (unlock_pup_dq_array[dq] == 0) { 1204 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - All pups are locked for DQ "); 1205 DEBUG_PBS_FULL_D(dq, 1); 1206 DEBUG_PBS_FULL_S("\n"); 1207 continue; 1208 } 1209 1210 for (pup = 0; pup < max_pup; pup++) { 1211 int idx; 1212 1213 idx = pup * (1 - ecc) + ecc * ECC_PUP; 1214 1215 if (IS_PUP_ACTIVE(unlock_pup_dq_array[dq], pup) 1216 == 0) 1217 continue; 1218 1219 if (is_tx == 1) 1220 ddr3_write_pup_reg( 1221 PUP_PBS_TX + pbs_dq_mapping[idx][dq], 1222 CS0, idx, 0, pbs_curr_val); 1223 else 1224 ddr3_write_pup_reg( 1225 PUP_PBS_RX + pbs_dq_mapping[idx][dq], 1226 CS0, idx, 0, pbs_curr_val); 1227 } 1228 } 1229 1230 /* 1231 * Write Read and compare results - run the test 1232 * DDR_PBS_COMP_RETRY_NUM times 1233 */ 1234 /* Run number of read and write to verify */ 1235 for (pbs_cmp_retry = 0; 1236 pbs_cmp_retry < COUNT_PBS_COMP_RETRY_NUM; 1237 pbs_cmp_retry++) { 1238 1239 if (MV_OK != 1240 ddr3_sdram_pbs_compare(dram_info, pup_locked, is_tx, 1241 pbs_pattern_idx, 1242 pbs_curr_val, start_pbs, 1243 skew_array, 1244 cmp_unlock_pup_dq_array 1245 [pbs_cmp_retry], ecc)) 1246 return MV_FAIL; 1247 1248 for (pup = 0; pup < max_pup; pup++) { 1249 for (dq = 0; dq < DQ_NUM; dq++) { 1250 if ((IS_PUP_ACTIVE(unlock_pup_dq_array[dq], 1251 pup) == 1) 1252 && (IS_PUP_ACTIVE(cmp_unlock_pup_dq_array 1253 [pbs_cmp_retry][dq], 1254 pup) == 0)) { 1255 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PbsCurrVal: "); 1256 DEBUG_PBS_FULL_D(pbs_curr_val, 2); 1257 DEBUG_PBS_FULL_S(" PUP: "); 1258 DEBUG_PBS_FULL_D(pup, 1); 1259 DEBUG_PBS_FULL_S(" DQ: "); 1260 DEBUG_PBS_FULL_D(dq, 1); 1261 DEBUG_PBS_FULL_S(" - failed\n"); 1262 } 1263 } 1264 } 1265 1266 for (dq = 0; dq < DQ_NUM; dq++) { 1267 unlock_pup_dq_array[dq] &= 1268 cmp_unlock_pup_dq_array[pbs_cmp_retry][dq]; 1269 } 1270 } 1271 1272 pup_locked = 0; 1273 sum_pup_fail = *pcur_pup; 1274 1275 /* Check which DQ is failed */ 1276 for (dq = 0; dq < DQ_NUM; dq++) { 1277 /* Summarize the locked pup */ 1278 pup_locked |= unlock_pup_dq_array[dq]; 1279 1280 /* Check if get fail */ 1281 sum_pup_fail &= unlock_pup_dq_array[dq]; 1282 } 1283 1284 /* If all PUPS are locked in all DQ - Break */ 1285 if (pup_locked == 0) { 1286 /* All pups are locked */ 1287 *start_over = 0; 1288 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - All bit in all pups are successfully locked\n"); 1289 break; 1290 } 1291 1292 /* PBS deskew elements reach max ? */ 1293 if (pbs_curr_val == last_pbs) { 1294 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - PBS deskew elements reach max\n"); 1295 /* CHIP-ONLY! - Implementation Limitation */ 1296 *start_over = (sum_pup_fail != 0) && (!(*start_over)); 1297 *pcur_pup = pup_locked; 1298 1299 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - StartOver: "); 1300 DEBUG_PBS_FULL_D(*start_over, 1); 1301 DEBUG_PBS_FULL_S(" pup_locked: "); 1302 DEBUG_PBS_FULL_D(pup_locked, 2); 1303 DEBUG_PBS_FULL_S(" sum_pup_fail: "); 1304 DEBUG_PBS_FULL_D(sum_pup_fail, 2); 1305 DEBUG_PBS_FULL_S("\n"); 1306 1307 /* Lock PBS value for all remaining bits */ 1308 for (pup = 0; pup < max_pup; pup++) { 1309 /* Check if current pup already received error */ 1310 if (IS_PUP_ACTIVE(pup_locked, pup) == 1) { 1311 /* Valid pup for current function */ 1312 if (IS_PUP_ACTIVE(sum_pup_fail, pup) == 1313 1 && (*start_over == 1)) { 1314 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - skipping lock of pup (first loop of pbs)", 1315 pup, 1); 1316 continue; 1317 } else 1318 if (IS_PUP_ACTIVE(sum_pup_fail, pup) 1319 == 1) { 1320 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - Locking pup %d (even though it wasn't supposed to be locked)", 1321 pup, 1); 1322 } 1323 1324 /* Already got fail on the PUP */ 1325 /* Lock PBS value for all remaining bits */ 1326 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Locking remaning DQs for pup - "); 1327 DEBUG_PBS_FULL_D(pup, 1); 1328 DEBUG_PBS_FULL_S(": "); 1329 1330 for (dq = 0; dq < DQ_NUM; dq++) { 1331 if (IS_PUP_ACTIVE 1332 (unlock_pup_dq_array[dq], 1333 pup) == 1) { 1334 DEBUG_PBS_FULL_D(dq, 1); 1335 DEBUG_PBS_FULL_S(","); 1336 /* set current PBS */ 1337 skew_array[((pup) * 1338 DQ_NUM) + 1339 dq] = 1340 pbs_curr_val; 1341 } 1342 } 1343 1344 if (*start_over == 1) { 1345 /* 1346 * Reset this pup bit - when 1347 * restart the PBS, ignore this 1348 * pup 1349 */ 1350 *pcur_pup &= ~(1 << pup); 1351 } 1352 DEBUG_PBS_FULL_S("\n"); 1353 } else { 1354 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - Pup "); 1355 DEBUG_PBS_FULL_D(pup, 1); 1356 DEBUG_PBS_FULL_C(" is not set in puplocked - ", 1357 pup_locked, 1); 1358 } 1359 } 1360 1361 /* Need to start the PBS again */ 1362 if (*start_over == 1) { 1363 DEBUG_PBS_FULL_S("DDR3 - PBS Per bit - false fail - returning to start\n"); 1364 return MV_OK; 1365 } 1366 break; 1367 } 1368 1369 /* Diff Check */ 1370 for (pup = 0; pup < max_pup; pup++) { 1371 if (IS_PUP_ACTIVE(pup_locked, pup) == 1) { 1372 /* pup is not locked */ 1373 if (first_failed[pup] == 0) { 1374 /* No first fail until now */ 1375 if (IS_PUP_ACTIVE(sum_pup_fail, pup) == 1376 0) { 1377 /* Get first fail */ 1378 DEBUG_PBS_FULL_C("DDR3 - PBS Per bit - First fail in pup ", 1379 pup, 1); 1380 first_failed[pup] = 1; 1381 first_fail[pup] = pbs_curr_val; 1382 } 1383 } else { 1384 /* Already got first fail */ 1385 if (is_tx == 1) { 1386 /* TX - inc pbs */ 1387 calc_pbs_diff = pbs_curr_val - 1388 first_fail[pup]; 1389 } else { 1390 /* RX - dec pbs */ 1391 calc_pbs_diff = first_fail[pup] - 1392 pbs_curr_val; 1393 } 1394 1395 if (calc_pbs_diff >= PBS_DIFF_LIMIT) { 1396 lock_pups(pup, &pup_locked, 1397 unlock_pup_dq_array, 1398 pbs_curr_val, 1399 start_pbs, ecc, is_tx); 1400 } 1401 } 1402 } 1403 } 1404 } while (1); 1405 1406 return MV_OK; 1407 } 1408 1409 /* 1410 * Name: ddr3_set_pbs_results 1411 * Desc: Set to HW the PBS phase results. 1412 * Args: is_tx Indicates whether to set Tx or RX results 1413 * Notes: 1414 * Returns: MV_OK if success, other error code if fail. 1415 */ 1416 static int ddr3_set_pbs_results(MV_DRAM_INFO *dram_info, int is_tx) 1417 { 1418 u32 pup, phys_pup, dq; 1419 u32 max_pup; /* number of valid pups */ 1420 u32 pbs_min; /* minimal pbs val per pup */ 1421 u32 pbs_max; /* maximum pbs val per pup */ 1422 u32 val[9]; 1423 1424 max_pup = dram_info->num_of_total_pups; 1425 DEBUG_PBS_FULL_S("DDR3 - PBS - ddr3_set_pbs_results:\n"); 1426 1427 /* Loop for all dqs & pups */ 1428 for (pup = 0; pup < max_pup; pup++) { 1429 if (pup == (max_pup - 1) && dram_info->ecc_ena) 1430 phys_pup = ECC_PUP; 1431 else 1432 phys_pup = pup; 1433 1434 /* 1435 * To minimize delay elements, inc from pbs value the min 1436 * pbs val 1437 */ 1438 pbs_min = MAX_PBS; 1439 pbs_max = 0; 1440 for (dq = 0; dq < DQ_NUM; dq++) { 1441 if (pbs_min > skew_array[(pup * DQ_NUM) + dq]) 1442 pbs_min = skew_array[(pup * DQ_NUM) + dq]; 1443 1444 if (pbs_max < skew_array[(pup * DQ_NUM) + dq]) 1445 pbs_max = skew_array[(pup * DQ_NUM) + dq]; 1446 } 1447 1448 pbs_max -= pbs_min; 1449 1450 DEBUG_PBS_FULL_S("DDR3 - PBS - PUP"); 1451 DEBUG_PBS_FULL_D(phys_pup, 1); 1452 DEBUG_PBS_FULL_S(": Min Val = "); 1453 DEBUG_PBS_FULL_D(pbs_min, 2); 1454 DEBUG_PBS_FULL_C(", Max Val = ", pbs_max, 2); 1455 1456 val[pup] = 0; 1457 1458 for (dq = 0; dq < DQ_NUM; dq++) { 1459 int idx; 1460 int offs; 1461 1462 /* Set skew value for all dq */ 1463 /* 1464 * Bit# Deskew <- Bit# Deskew - last / first 1465 * failing bit Deskew For all bits (per PUP) 1466 * (minimize delay elements) 1467 */ 1468 1469 DEBUG_PBS_FULL_S("DQ"); 1470 DEBUG_PBS_FULL_D(dq, 1); 1471 DEBUG_PBS_FULL_S("-"); 1472 DEBUG_PBS_FULL_D((skew_array[(pup * DQ_NUM) + dq] - 1473 pbs_min), 2); 1474 DEBUG_PBS_FULL_S(", "); 1475 1476 idx = (pup * DQ_NUM) + dq; 1477 1478 if (is_tx == 1) 1479 offs = PUP_PBS_TX; 1480 else 1481 offs = PUP_PBS_RX; 1482 1483 ddr3_write_pup_reg(offs + pbs_dq_mapping[phys_pup][dq], 1484 CS0, phys_pup, 0, 1485 skew_array[idx] - pbs_min); 1486 1487 if (is_tx == 1) 1488 val[pup] += skew_array[idx] - pbs_min; 1489 } 1490 1491 DEBUG_PBS_FULL_S("\n"); 1492 1493 /* Set the DQS the half of the Max PBS of the DQs */ 1494 if (is_tx == 1) { 1495 ddr3_write_pup_reg(PUP_PBS_TX + 8, CS0, phys_pup, 0, 1496 pbs_max / 2); 1497 ddr3_write_pup_reg(PUP_PBS_TX + 0xa, CS0, phys_pup, 0, 1498 val[pup] / 8); 1499 } else 1500 ddr3_write_pup_reg(PUP_PBS_RX + 8, CS0, phys_pup, 0, 1501 pbs_max / 2); 1502 } 1503 1504 return MV_OK; 1505 } 1506 1507 static void ddr3_pbs_write_pup_dqs_reg(u32 cs, u32 pup, u32 dqs_delay) 1508 { 1509 u32 reg, delay; 1510 1511 reg = (ddr3_read_pup_reg(PUP_WL_MODE, cs, pup) & 0x3FF); 1512 delay = reg & PUP_DELAY_MASK; 1513 reg |= ((dqs_delay + delay) << REG_PHY_DQS_REF_DLY_OFFS); 1514 reg |= REG_PHY_REGISTRY_FILE_ACCESS_OP_WR; 1515 reg |= (pup << REG_PHY_PUP_OFFS); 1516 reg |= ((0x4 * cs + PUP_WL_MODE) << REG_PHY_CS_OFFS); 1517 1518 reg_write(REG_PHY_REGISTRY_FILE_ACCESS_ADDR, reg); /* 0x16A0 */ 1519 do { 1520 reg = reg_read(REG_PHY_REGISTRY_FILE_ACCESS_ADDR) & 1521 REG_PHY_REGISTRY_FILE_ACCESS_OP_DONE; 1522 } while (reg); /* Wait for '0' to mark the end of the transaction */ 1523 1524 udelay(10); 1525 } 1526 1527 /* 1528 * Set training patterns 1529 */ 1530 int ddr3_load_pbs_patterns(MV_DRAM_INFO *dram_info) 1531 { 1532 u32 cs, cs_count, cs_tmp; 1533 u32 sdram_addr; 1534 u32 *pattern_ptr0, *pattern_ptr1; 1535 1536 /* Choose pattern */ 1537 switch (dram_info->ddr_width) { 1538 #if defined(MV88F672X) 1539 case 16: 1540 pattern_ptr0 = (u32 *)&pbs_pattern[0]; 1541 pattern_ptr1 = (u32 *)&pbs_pattern[1]; 1542 break; 1543 #endif 1544 case 32: 1545 pattern_ptr0 = (u32 *)&pbs_pattern_32b[0]; 1546 pattern_ptr1 = (u32 *)&pbs_pattern_32b[1]; 1547 break; 1548 #if defined(MV88F78X60) 1549 case 64: 1550 pattern_ptr0 = (u32 *)&pbs_pattern_64b[0]; 1551 pattern_ptr1 = (u32 *)&pbs_pattern_64b[1]; 1552 break; 1553 #endif 1554 default: 1555 return MV_FAIL; 1556 } 1557 1558 /* Loop for each CS */ 1559 for (cs = 0; cs < MAX_CS; cs++) { 1560 if (dram_info->cs_ena & (1 << cs)) { 1561 cs_count = 0; 1562 for (cs_tmp = 0; cs_tmp < cs; cs_tmp++) { 1563 if (dram_info->cs_ena & (1 << cs_tmp)) 1564 cs_count++; 1565 } 1566 1567 /* Init PBS I pattern */ 1568 sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) + 1569 SDRAM_PBS_I_OFFS); 1570 if (MV_OK != 1571 ddr3_sdram_compare(dram_info, (u32) NULL, NULL, 1572 pattern_ptr0, LEN_STD_PATTERN, 1573 sdram_addr, 1, 0, NULL, 1574 0)) 1575 return MV_FAIL; 1576 1577 /* Init PBS II pattern */ 1578 sdram_addr = (cs_count * (SDRAM_CS_SIZE + 1) + 1579 SDRAM_PBS_II_OFFS); 1580 if (MV_OK != 1581 ddr3_sdram_compare(dram_info, (u32) NULL, NULL, 1582 pattern_ptr1, LEN_STD_PATTERN, 1583 sdram_addr, 1, 0, NULL, 1584 0)) 1585 return MV_FAIL; 1586 } 1587 } 1588 1589 return MV_OK; 1590 } 1591 #endif 1592
