1 /* 2 * Copyright 2020-2025 NXP 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 #include <errno.h> 7 8 #include <assert.h> 9 #include <common/debug.h> 10 #include <ddr_utils.h> 11 #include <mmio_poll.h> 12 13 static uint32_t get_mail(uint32_t *mail); 14 static uint32_t ack_mail(void); 15 static uint8_t get_max_cdd(const uint32_t cdd_addr[], size_t size); 16 static uint16_t get_max_delay(const uint32_t delay_addr[], size_t size); 17 static uint8_t get_avg_vref(const uint32_t vref_addr[], size_t size); 18 static bool is_lpddr4(void); 19 20 static struct space_timing_params tr_res = { 21 .cdd = {.rr = 0, .rw = 0, .wr = 0, .ww = 0}, 22 .vref_ca = 0, 23 .vref_dq = 0, 24 .tphy_wrdata_delay = 0 25 }; 26 27 /* Modify bitfield value with delta, given bitfield position and mask */ 28 bool update_bf(uint32_t *v, uint8_t pos, uint32_t mask, int32_t delta) 29 { 30 uint32_t bf_val; 31 int64_t new_val; 32 33 bf_val = (*v >> pos) & mask; 34 new_val = (int64_t)bf_val + delta; 35 36 /* Check if new value is within valid range [0, mask] */ 37 if ((new_val < 0) || (new_val > (int64_t)mask)) { 38 return false; 39 } 40 41 *v = (*v & ~(mask << pos)) | ((uint32_t)new_val << pos); 42 return true; 43 } 44 45 /* Sets default AXI parity. */ 46 uint32_t set_axi_parity(void) 47 { 48 uint32_t swstat_reg, timeout = DEFAULT_TIMEOUT_US; 49 int err; 50 51 /* Enable Parity For All AXI Interfaces */ 52 mmio_setbits_32(DDR_SS_REG, DDR_SS_AXI_PARITY_ENABLE_MASK); 53 54 /* Set AXI_PARITY_TYPE to 0x1ff; 0-even, 1-odd */ 55 mmio_setbits_32(DDR_SS_REG, DDR_SS_AXI_PARITY_TYPE_MASK); 56 57 /* For LPDDR4 Set DFI1_ENABLED to 0x1 */ 58 if (is_lpddr4()) { 59 mmio_setbits_32(DDR_SS_REG, DDR_SS_DFI_1_ENABLED); 60 } 61 62 /* Enable HIF, CAM Queueing */ 63 mmio_write_32(DDRC_BASE + OFFSET_DDRC_DBG1, DBG1_DISABLE_DE_QUEUEING); 64 65 /* Disable auto-refresh: RFSHCTL3.dis_auto_refresh = 1 */ 66 mmio_setbits_32(DDRC_BASE + OFFSET_DDRC_RFSHCTL3, RFSHCTL3_DISABLE_AUTO_REFRESH); 67 68 /* Disable power down: PWRCTL.powerdown_en = 0 */ 69 mmio_clrbits_32(DDRC_BASE + OFFSET_DDRC_PWRCTL, PWRCTL_POWER_DOWN_ENABLE_MASK); 70 71 /* Disable self-refresh: PWRCTL.selfref_en = 0 */ 72 mmio_clrbits_32(DDRC_BASE + OFFSET_DDRC_PWRCTL, PWRCTL_SELF_REFRESH_ENABLE_MASK); 73 74 /* 75 * Disable assertion of dfi_dram_clk_disable: 76 * PWRTL.en_dfi_dram_clk_disable = 0 77 */ 78 mmio_clrbits_32(DDRC_BASE + OFFSET_DDRC_PWRCTL, PWRCTL_EN_DFI_DRAM_CLOCK_DIS_MASK); 79 80 /* Enable Quasi-Dynamic Programming */ 81 mmio_write_32(DDRC_BASE + OFFSET_DDRC_SWCTL, SWCTL_SWDONE_ENABLE); 82 83 /* Confirm Register Programming Done Ack is Cleared */ 84 err = mmio_read_32_poll_timeout(DDRC_BASE + OFFSET_DDRC_SWSTAT, swstat_reg, 85 (swstat_reg & SWSTAT_SWDONE_ACK_MASK) != SWSTAT_SW_DONE, 86 timeout); 87 if (err != 0) { 88 ERROR("Failed to clear register programming done ACK\n"); 89 return TIMEOUT_ERR; 90 } 91 92 /* DFI_INIT_COMPLETE_EN set to 0 */ 93 mmio_clrbits_32(DDRC_BASE + OFFSET_DDRC_DFIMISC, DFIMISC_DFI_INIT_COMPLETE_EN_MASK); 94 95 /* Set SWCTL.sw_done to 1 */ 96 mmio_write_32(DDRC_BASE + OFFSET_DDRC_SWCTL, SWCTL_SWDONE_DONE); 97 98 err = mmio_read_32_poll_timeout(DDRC_BASE + OFFSET_DDRC_SWSTAT, swstat_reg, 99 (swstat_reg & SWSTAT_SWDONE_ACK_MASK) != SWSTAT_SW_NOT_DONE, 100 timeout); 101 if (err != 0) { 102 ERROR("Failed to confirm DDRC SWSTAT switch done ACK\n"); 103 return TIMEOUT_ERR; 104 } 105 106 return NO_ERR; 107 } 108 109 /* Wait until firmware finishes execution and return training result */ 110 uint32_t wait_firmware_execution(void) 111 { 112 uint32_t timeout_us = DEFAULT_TIMEOUT_US, ret = NO_ERR, mail = 0; 113 uint64_t timeout = timeout_init_us(timeout_us); 114 bool loop_continue = true; 115 bool timeout_expired; 116 117 do { 118 ret = get_mail(&mail); 119 if (ret != NO_ERR) { 120 loop_continue = false; 121 } else if (mail == TRAINING_FAILED_MSG) { 122 /* Training stage failed */ 123 ret = TRAINING_FAILED; 124 loop_continue = false; 125 } else if (mail == TRAINING_OK_MSG) { 126 loop_continue = false; 127 } else { 128 /* Continue waiting for training result */ 129 } 130 timeout_expired = timeout_elapsed(timeout); 131 if (timeout_expired) { 132 ret = TRAINING_FAILED; 133 loop_continue = false; 134 } 135 /* Continue loop if no exit condition met and timeout not elapsed */ 136 } while (loop_continue); 137 138 return ret; 139 } 140 141 /* Acknowledge received message */ 142 static uint32_t ack_mail(void) 143 { 144 uint32_t timeout = DEFAULT_TIMEOUT_US; 145 uint32_t uct_reg; 146 int err; 147 148 /* ACK message */ 149 mmio_write_32(DDR_PHYA_DCTWRITEPROT, APBONLY_DCTWRITEPROT_ACK_EN); 150 151 err = mmio_read_32_poll_timeout(DDR_PHYA_APBONLY_UCTSHADOWREGS, uct_reg, 152 (uct_reg & UCT_WRITE_PROT_SHADOW_MASK) != 153 UCT_WRITE_PROT_SHADOW_ACK, 154 timeout); 155 if (err != 0) { 156 ERROR("DDR PHY did not acknowledge write protection\n"); 157 return TIMEOUT_ERR; 158 } 159 160 mmio_write_32(DDR_PHYA_DCTWRITEPROT, APBONLY_DCTWRITEPROT_ACK_DIS); 161 162 return NO_ERR; 163 } 164 165 /* Read available message from DDR PHY microcontroller */ 166 static uint32_t get_mail(uint32_t *mail) 167 { 168 uint32_t uct_reg, timeout = DEFAULT_TIMEOUT_US; 169 int err; 170 171 err = mmio_read_32_poll_timeout(DDR_PHYA_APBONLY_UCTSHADOWREGS, uct_reg, 172 (uct_reg & UCT_WRITE_PROT_SHADOW_MASK) == 173 UCT_WRITE_PROT_SHADOW_ACK, 174 timeout); 175 if (err != 0) { 176 ERROR("DDR PHY did not acknowledge UCT write protection\n"); 177 return TIMEOUT_ERR; 178 } 179 180 *mail = mmio_read_32(DDR_PHYA_APBONLY_UCTWRITEONLYSHADOW); 181 /* ACK */ 182 return ack_mail(); 183 } 184 185 /* Read Critical Delay Differences from message block and store max values */ 186 void read_cdds(void) 187 { 188 const uint32_t rank0_rw_addr[] = {CDD_CHA_RW_0_0, CDD_CHB_RW_0_0}; 189 const uint32_t rank0_wr_addr[] = {CDD_CHA_WR_0_0, CDD_CHB_WR_0_0}; 190 uint8_t cdd_rr = 0, cdd_ww = 0, cdd_wr = 0, cdd_rw = 0; 191 uint32_t mstr; 192 193 /* Max CDD values for single-rank */ 194 tr_res.cdd.rr = cdd_rr; 195 tr_res.cdd.ww = cdd_ww; 196 tr_res.cdd.rw = is_lpddr4() ? 197 get_max_cdd(rank0_rw_addr, ARRAY_SIZE(rank0_rw_addr)) : 198 mmio_read_8(CDD_CHA_RW_0_0_DDR3); 199 tr_res.cdd.wr = is_lpddr4() ? 200 get_max_cdd(rank0_wr_addr, ARRAY_SIZE(rank0_wr_addr)) : 201 mmio_read_8(CDD_CHA_WR_0_0_DDR3); 202 203 /* Check MSTR.active_ranks to identify multi-rank configurations */ 204 mstr = mmio_read_32(DDRC_BASE); 205 if ((mstr & MSTR_ACT_RANKS_MASK) == MSTR_DUAL_RANK_VAL) { 206 /* Compute max CDDs for both ranks depending on memory type */ 207 if (is_lpddr4()) { 208 const uint32_t rr_addr[] = { 209 CDD_CHA_RR_1_0, CDD_CHA_RR_0_1, 210 CDD_CHB_RR_1_0, CDD_CHB_RR_0_1 211 }; 212 const uint32_t ww_addr[] = { 213 CDD_CHA_WW_1_0, CDD_CHA_WW_0_1, 214 CDD_CHB_WW_1_0, CDD_CHB_WW_0_1 215 }; 216 const uint32_t rw_addr[] = { 217 CDD_CHA_RW_1_1, CDD_CHA_RW_1_0, 218 CDD_CHA_RW_0_1, CDD_CHB_RW_1_1, 219 CDD_CHB_RW_1_0, CDD_CHB_RW_0_1 220 }; 221 const uint32_t wr_addr[] = { 222 CDD_CHA_WR_1_1, CDD_CHA_WR_1_0, 223 CDD_CHA_WR_0_1, CDD_CHB_WR_1_1, 224 CDD_CHB_WR_1_0, CDD_CHB_WR_0_1 225 }; 226 227 cdd_rr = get_max_cdd(rr_addr, ARRAY_SIZE(rr_addr)); 228 cdd_rw = get_max_cdd(rw_addr, ARRAY_SIZE(rw_addr)); 229 cdd_wr = get_max_cdd(wr_addr, ARRAY_SIZE(wr_addr)); 230 cdd_ww = get_max_cdd(ww_addr, ARRAY_SIZE(ww_addr)); 231 } else { 232 const uint32_t rr_addr[] = {CDD_CHA_RR_1_0_DDR3, 233 CDD_CHA_RR_0_1_DDR3}; 234 const uint32_t ww_addr[] = {CDD_CHA_WW_1_0_DDR3, 235 CDD_CHA_WW_0_1_DDR3}; 236 const uint32_t rw_addr[] = {CDD_CHA_RW_1_1_DDR3, 237 CDD_CHA_RW_1_0_DDR3, 238 CDD_CHA_RW_0_1_DDR3}; 239 const uint32_t wr_addr[] = {CDD_CHA_WR_1_1_DDR3, 240 CDD_CHA_WR_1_0_DDR3, 241 CDD_CHA_WR_0_1_DDR3}; 242 243 cdd_rr = get_max_cdd(rr_addr, ARRAY_SIZE(rr_addr)); 244 cdd_rw = get_max_cdd(rw_addr, ARRAY_SIZE(rw_addr)); 245 cdd_wr = get_max_cdd(wr_addr, ARRAY_SIZE(wr_addr)); 246 cdd_ww = get_max_cdd(ww_addr, ARRAY_SIZE(ww_addr)); 247 } 248 249 /* Update max CDD values if needed */ 250 if (cdd_rr > tr_res.cdd.rr) { 251 tr_res.cdd.rr = cdd_rr; 252 } 253 if (cdd_rw > tr_res.cdd.rw) { 254 tr_res.cdd.rw = cdd_rw; 255 } 256 if (cdd_wr > tr_res.cdd.wr) { 257 tr_res.cdd.wr = cdd_wr; 258 } 259 if (cdd_ww > tr_res.cdd.ww) { 260 tr_res.cdd.ww = cdd_ww; 261 } 262 } 263 } 264 265 /* Read trained VrefCA from message block and store average value */ 266 void read_vref_ca(void) 267 { 268 const uint32_t rank0_vref_addr[] = {VREF_CA_A0, VREF_CA_B0}; 269 const uint32_t rank01_vref_addr[] = {VREF_CA_A0, VREF_CA_A1, 270 VREF_CA_B0, VREF_CA_B1}; 271 uint32_t mstr; 272 273 /* Check MSTR.active_ranks to identify multi-rank configurations */ 274 mstr = mmio_read_32(DDRC_BASE); 275 if ((mstr & MSTR_ACT_RANKS_MASK) == MSTR_DUAL_RANK_VAL) { 276 tr_res.vref_ca = get_avg_vref(rank01_vref_addr, 277 ARRAY_SIZE(rank01_vref_addr)); 278 } else { 279 tr_res.vref_ca = get_avg_vref(rank0_vref_addr, 280 ARRAY_SIZE(rank0_vref_addr)); 281 } 282 } 283 284 /* Read trained VrefDQ from message block and store average value*/ 285 void read_vref_dq(void) 286 { 287 const uint32_t rank0_vref_addr[] = {VREF_DQ_A0, VREF_DQ_B0}; 288 const uint32_t rank01_vref_addr[] = {VREF_DQ_A0, VREF_DQ_A1, 289 VREF_DQ_B0, VREF_DQ_B1}; 290 uint32_t mstr; 291 292 /* Check MSTR.active_ranks to identify multi-rank configurations */ 293 mstr = mmio_read_32(DDRC_BASE); 294 if ((mstr & MSTR_ACT_RANKS_MASK) == MSTR_DUAL_RANK_VAL) { 295 tr_res.vref_dq = get_avg_vref(rank01_vref_addr, 296 ARRAY_SIZE(rank01_vref_addr)); 297 } else { 298 tr_res.vref_dq = get_avg_vref(rank0_vref_addr, 299 ARRAY_SIZE(rank0_vref_addr)); 300 } 301 } 302 303 /* Calculate DFITMG1.dfi_t_wrdata_delay */ 304 void compute_tphy_wrdata_delay(void) 305 { 306 uint16_t tx_dqsdly, tx_dqsdly_tg1, tctrl_delay, burst_length, 307 wrdata_use_dfi_phy_clk; 308 309 const uint32_t single_rank_dly_addr[] = { 310 DBYTE0_TXDQSDLYTG0_U0, DBYTE0_TXDQSDLYTG0_U1, 311 DBYTE1_TXDQSDLYTG0_U0, DBYTE1_TXDQSDLYTG0_U1, 312 DBYTE2_TXDQSDLYTG0_U0, DBYTE2_TXDQSDLYTG0_U1, 313 DBYTE3_TXDQSDLYTG0_U0, DBYTE3_TXDQSDLYTG0_U1 314 }; 315 316 const uint32_t dual_rank_dly_addr[] = { 317 DBYTE0_TXDQSDLYTG1_U0, DBYTE0_TXDQSDLYTG1_U1, 318 DBYTE1_TXDQSDLYTG1_U0, DBYTE1_TXDQSDLYTG1_U1, 319 DBYTE2_TXDQSDLYTG1_U0, DBYTE2_TXDQSDLYTG1_U1, 320 DBYTE3_TXDQSDLYTG1_U0, DBYTE3_TXDQSDLYTG1_U1 321 }; 322 323 uint32_t mstr, dfitmg0; 324 325 /* Compute max tx_dqdqsdly for rank 0 */ 326 tx_dqsdly = get_max_delay(single_rank_dly_addr, 327 ARRAY_SIZE(single_rank_dly_addr)); 328 329 /* Check MSTR.active_ranks to identify multi-rank configurations */ 330 mstr = mmio_read_32(DDRC_BASE); 331 if ((mstr & MSTR_ACT_RANKS_MASK) == MSTR_DUAL_RANK_VAL) { 332 /* Compute max tx_dqdqsdly for rank 1 */ 333 tx_dqsdly_tg1 = get_max_delay(dual_rank_dly_addr, 334 ARRAY_SIZE(dual_rank_dly_addr)); 335 if (tx_dqsdly_tg1 > tx_dqsdly) { 336 tx_dqsdly = tx_dqsdly_tg1; 337 } 338 } 339 340 /* Extract coarse delay value + 1 for fine delay */ 341 tx_dqsdly = (tx_dqsdly >> TXDQDLY_COARSE) + 1U; 342 343 /* Compute tctrl_delay */ 344 tctrl_delay = (uint16_t)((mmio_read_16(ARDPTR_INITVAL_ADDR) / 2U) + 345 (DDRPHY_PIPE_DFI_MISC * 2U) + 3U); 346 347 burst_length = (uint16_t)(mstr >> MSTR_BURST_RDWR_POS) & 348 MSTR_BURST_RDWR_MASK; 349 dfitmg0 = mmio_read_16(DDRC_BASE + OFFSET_DDRC_DFITMG0); 350 wrdata_use_dfi_phy_clk = (uint16_t)(dfitmg0 >> DFITMG0_PHY_CLK_POS) & 351 DFITMG0_PHY_CLK_MASK; 352 353 /* Program */ 354 tr_res.tphy_wrdata_delay = tctrl_delay + 6U + burst_length + 355 wrdata_use_dfi_phy_clk + tx_dqsdly; 356 tr_res.tphy_wrdata_delay = (tr_res.tphy_wrdata_delay / 2U) + 357 (tr_res.tphy_wrdata_delay % 2U); 358 } 359 360 /* Check if memory type is LPDDR4 using MSTR register */ 361 static bool is_lpddr4(void) 362 { 363 uint32_t mstr; 364 365 mstr = mmio_read_32(DDRC_BASE); 366 return ((mstr & MSTR_DRAM_MASK) == MSTR_LPDDR4_VAL); 367 } 368 369 /* 370 * Get maximum critical delay difference value. 371 * @param cdd_addr[] - list of CDD memory addresses 372 * @param size - number of CDDs to be read 373 * @return max CDD value 374 */ 375 static uint8_t get_max_cdd(const uint32_t cdd_addr[], size_t size) 376 { 377 uint8_t cdd, max = 0; 378 int8_t signed_cdd; 379 size_t i; 380 381 for (i = 0; i < size; i++) { 382 /* CDD has type int8_t - read as unsigned and cast to signed */ 383 signed_cdd = (int8_t)(mmio_read_8(cdd_addr[i])); 384 /* We need to use absolute value */ 385 cdd = (uint8_t)((signed_cdd >= 0) ? signed_cdd : -signed_cdd); 386 max = MAX(cdd, max); 387 } 388 return max; 389 } 390 391 /* 392 * Get maximum delay value. 393 * @param delay_addr[] - list of CDD memory addresses 394 * @param size - number of values to be read 395 * @return max delay value 396 */ 397 static uint16_t get_max_delay(const uint32_t delay_addr[], size_t size) 398 { 399 uint16_t value, max = 0; 400 size_t i; 401 402 for (i = 0; i < size; i++) { 403 value = mmio_read_16(delay_addr[i]); 404 max = MAX(value, max); 405 } 406 return max; 407 } 408 409 /* 410 * Compute average vref value. 411 * @param vref_addr[] - list of vref memory addresses 412 * @param size - number of values to be read 413 * @return average vref value 414 */ 415 static uint8_t get_avg_vref(const uint32_t vref_addr[], size_t size) 416 { 417 uint32_t sum = 0; 418 size_t i; 419 420 for (i = 0; i < size; i++) { 421 sum += mmio_read_8(vref_addr[i]); 422 } 423 424 assert((sum / size) <= UINT8_MAX); 425 426 return (uint8_t)(sum / size); 427 } 428