1 /* 2 * Copyright 2020-2026 NXP 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 #include <errno.h> 7 8 #include <assert.h> 9 #include <common/debug.h> 10 #include <ddr_utils.h> 11 #include <mmio_poll.h> 12 #include <s32cc-clk-drv.h> 13 14 static uint32_t get_mail(uint32_t *mail); 15 static uint32_t ack_mail(void); 16 static uint8_t get_max_cdd(const uint32_t cdd_addr[], size_t size); 17 static uint16_t get_max_delay(const uint32_t delay_addr[], size_t size); 18 static uint8_t get_avg_vref(const uint32_t vref_addr[], size_t size); 19 static bool is_lpddr4(void); 20 21 static struct space_timing_params tr_res = { 22 .cdd = {.rr = 0, .rw = 0, .wr = 0, .ww = 0}, 23 .vref_ca = 0, 24 .vref_dq = 0, 25 .tphy_wrdata_delay = 0 26 }; 27 28 /* Modify bitfield value with delta, given bitfield position and mask */ 29 bool update_bf(uint32_t *v, uint8_t pos, uint32_t mask, int32_t delta) 30 { 31 uint32_t bf_val; 32 int64_t new_val; 33 34 bf_val = (*v >> pos) & mask; 35 new_val = (int64_t)bf_val + delta; 36 37 /* Check if new value is within valid range [0, mask] */ 38 if ((new_val < 0) || (new_val > (int64_t)mask)) { 39 return false; 40 } 41 42 *v = (*v & ~(mask << pos)) | ((uint32_t)new_val << pos); 43 return true; 44 } 45 46 /* Sets default AXI parity. */ 47 uint32_t set_axi_parity(void) 48 { 49 uint32_t swstat_reg, timeout = DEFAULT_TIMEOUT_US; 50 int err; 51 52 /* Enable Parity For All AXI Interfaces */ 53 mmio_setbits_32(DDR_SS_REG, DDR_SS_AXI_PARITY_ENABLE_MASK); 54 55 /* Set AXI_PARITY_TYPE to 0x1ff; 0-even, 1-odd */ 56 mmio_setbits_32(DDR_SS_REG, DDR_SS_AXI_PARITY_TYPE_MASK); 57 58 /* For LPDDR4 Set DFI1_ENABLED to 0x1 */ 59 if (is_lpddr4()) { 60 mmio_setbits_32(DDR_SS_REG, DDR_SS_DFI_1_ENABLED); 61 } 62 63 if (plat_deassert_ddr_reset() != 0) { 64 return DEASSERT_FAILED; 65 } 66 67 /* Enable HIF, CAM Queueing */ 68 mmio_write_32(DDRC_BASE + OFFSET_DDRC_DBG1, DBG1_DISABLE_DE_QUEUEING); 69 70 /* Disable auto-refresh: RFSHCTL3.dis_auto_refresh = 1 */ 71 mmio_setbits_32(DDRC_BASE + OFFSET_DDRC_RFSHCTL3, RFSHCTL3_DISABLE_AUTO_REFRESH); 72 73 /* Disable power down: PWRCTL.powerdown_en = 0 */ 74 mmio_clrbits_32(DDRC_BASE + OFFSET_DDRC_PWRCTL, PWRCTL_POWER_DOWN_ENABLE_MASK); 75 76 /* Disable self-refresh: PWRCTL.selfref_en = 0 */ 77 mmio_clrbits_32(DDRC_BASE + OFFSET_DDRC_PWRCTL, PWRCTL_SELF_REFRESH_ENABLE_MASK); 78 79 /* 80 * Disable assertion of dfi_dram_clk_disable: 81 * PWRTL.en_dfi_dram_clk_disable = 0 82 */ 83 mmio_clrbits_32(DDRC_BASE + OFFSET_DDRC_PWRCTL, PWRCTL_EN_DFI_DRAM_CLOCK_DIS_MASK); 84 85 /* Enable Quasi-Dynamic Programming */ 86 mmio_write_32(DDRC_BASE + OFFSET_DDRC_SWCTL, SWCTL_SWDONE_ENABLE); 87 88 /* Confirm Register Programming Done Ack is Cleared */ 89 err = mmio_read_32_poll_timeout(DDRC_BASE + OFFSET_DDRC_SWSTAT, swstat_reg, 90 (swstat_reg & SWSTAT_SWDONE_ACK_MASK) != SWSTAT_SW_DONE, 91 timeout); 92 if (err != 0) { 93 ERROR("Failed to clear register programming done ACK\n"); 94 return TIMEOUT_ERR; 95 } 96 97 /* DFI_INIT_COMPLETE_EN set to 0 */ 98 mmio_clrbits_32(DDRC_BASE + OFFSET_DDRC_DFIMISC, DFIMISC_DFI_INIT_COMPLETE_EN_MASK); 99 100 /* Set SWCTL.sw_done to 1 */ 101 mmio_write_32(DDRC_BASE + OFFSET_DDRC_SWCTL, SWCTL_SWDONE_DONE); 102 103 err = mmio_read_32_poll_timeout(DDRC_BASE + OFFSET_DDRC_SWSTAT, swstat_reg, 104 (swstat_reg & SWSTAT_SWDONE_ACK_MASK) != SWSTAT_SW_NOT_DONE, 105 timeout); 106 if (err != 0) { 107 ERROR("Failed to confirm DDRC SWSTAT switch done ACK\n"); 108 return TIMEOUT_ERR; 109 } 110 111 return NO_ERR; 112 } 113 114 /* Wait until firmware finishes execution and return training result */ 115 uint32_t wait_firmware_execution(void) 116 { 117 uint32_t timeout_us = DEFAULT_TIMEOUT_US, ret = NO_ERR, mail = 0; 118 uint64_t timeout = timeout_init_us(timeout_us); 119 bool loop_continue = true; 120 bool timeout_expired; 121 122 do { 123 ret = get_mail(&mail); 124 if (ret != NO_ERR) { 125 loop_continue = false; 126 } else if (mail == TRAINING_FAILED_MSG) { 127 /* Training stage failed */ 128 ret = TRAINING_FAILED; 129 loop_continue = false; 130 } else if (mail == TRAINING_OK_MSG) { 131 loop_continue = false; 132 } else { 133 /* Continue waiting for training result */ 134 } 135 timeout_expired = timeout_elapsed(timeout); 136 if (timeout_expired) { 137 ret = TRAINING_FAILED; 138 loop_continue = false; 139 } 140 /* Continue loop if no exit condition met and timeout not elapsed */ 141 } while (loop_continue); 142 143 return ret; 144 } 145 146 /* Acknowledge received message */ 147 static uint32_t ack_mail(void) 148 { 149 uint32_t timeout = DEFAULT_TIMEOUT_US; 150 uint32_t uct_reg; 151 int err; 152 153 /* ACK message */ 154 mmio_write_32(DDR_PHYA_DCTWRITEPROT, APBONLY_DCTWRITEPROT_ACK_EN); 155 156 err = mmio_read_32_poll_timeout(DDR_PHYA_APBONLY_UCTSHADOWREGS, uct_reg, 157 (uct_reg & UCT_WRITE_PROT_SHADOW_MASK) != 158 UCT_WRITE_PROT_SHADOW_ACK, 159 timeout); 160 if (err != 0) { 161 ERROR("DDR PHY did not acknowledge write protection\n"); 162 return TIMEOUT_ERR; 163 } 164 165 mmio_write_32(DDR_PHYA_DCTWRITEPROT, APBONLY_DCTWRITEPROT_ACK_DIS); 166 167 return NO_ERR; 168 } 169 170 /* Read available message from DDR PHY microcontroller */ 171 static uint32_t get_mail(uint32_t *mail) 172 { 173 uint32_t uct_reg, timeout = DEFAULT_TIMEOUT_US; 174 int err; 175 176 err = mmio_read_32_poll_timeout(DDR_PHYA_APBONLY_UCTSHADOWREGS, uct_reg, 177 (uct_reg & UCT_WRITE_PROT_SHADOW_MASK) == 178 UCT_WRITE_PROT_SHADOW_ACK, 179 timeout); 180 if (err != 0) { 181 ERROR("DDR PHY did not acknowledge UCT write protection\n"); 182 return TIMEOUT_ERR; 183 } 184 185 *mail = mmio_read_32(DDR_PHYA_APBONLY_UCTWRITEONLYSHADOW); 186 /* ACK */ 187 return ack_mail(); 188 } 189 190 /* Read Critical Delay Differences from message block and store max values */ 191 void read_cdds(void) 192 { 193 const uint32_t rank0_rw_addr[] = {CDD_CHA_RW_0_0, CDD_CHB_RW_0_0}; 194 const uint32_t rank0_wr_addr[] = {CDD_CHA_WR_0_0, CDD_CHB_WR_0_0}; 195 uint8_t cdd_rr = 0, cdd_ww = 0, cdd_wr = 0, cdd_rw = 0; 196 uint32_t mstr; 197 198 /* Max CDD values for single-rank */ 199 tr_res.cdd.rr = cdd_rr; 200 tr_res.cdd.ww = cdd_ww; 201 tr_res.cdd.rw = is_lpddr4() ? 202 get_max_cdd(rank0_rw_addr, ARRAY_SIZE(rank0_rw_addr)) : 203 mmio_read_8(CDD_CHA_RW_0_0_DDR3); 204 tr_res.cdd.wr = is_lpddr4() ? 205 get_max_cdd(rank0_wr_addr, ARRAY_SIZE(rank0_wr_addr)) : 206 mmio_read_8(CDD_CHA_WR_0_0_DDR3); 207 208 /* Check MSTR.active_ranks to identify multi-rank configurations */ 209 mstr = mmio_read_32(DDRC_BASE); 210 if ((mstr & MSTR_ACT_RANKS_MASK) == MSTR_DUAL_RANK_VAL) { 211 /* Compute max CDDs for both ranks depending on memory type */ 212 if (is_lpddr4()) { 213 const uint32_t rr_addr[] = { 214 CDD_CHA_RR_1_0, CDD_CHA_RR_0_1, 215 CDD_CHB_RR_1_0, CDD_CHB_RR_0_1 216 }; 217 const uint32_t ww_addr[] = { 218 CDD_CHA_WW_1_0, CDD_CHA_WW_0_1, 219 CDD_CHB_WW_1_0, CDD_CHB_WW_0_1 220 }; 221 const uint32_t rw_addr[] = { 222 CDD_CHA_RW_1_1, CDD_CHA_RW_1_0, 223 CDD_CHA_RW_0_1, CDD_CHB_RW_1_1, 224 CDD_CHB_RW_1_0, CDD_CHB_RW_0_1 225 }; 226 const uint32_t wr_addr[] = { 227 CDD_CHA_WR_1_1, CDD_CHA_WR_1_0, 228 CDD_CHA_WR_0_1, CDD_CHB_WR_1_1, 229 CDD_CHB_WR_1_0, CDD_CHB_WR_0_1 230 }; 231 232 cdd_rr = get_max_cdd(rr_addr, ARRAY_SIZE(rr_addr)); 233 cdd_rw = get_max_cdd(rw_addr, ARRAY_SIZE(rw_addr)); 234 cdd_wr = get_max_cdd(wr_addr, ARRAY_SIZE(wr_addr)); 235 cdd_ww = get_max_cdd(ww_addr, ARRAY_SIZE(ww_addr)); 236 } else { 237 const uint32_t rr_addr[] = {CDD_CHA_RR_1_0_DDR3, 238 CDD_CHA_RR_0_1_DDR3}; 239 const uint32_t ww_addr[] = {CDD_CHA_WW_1_0_DDR3, 240 CDD_CHA_WW_0_1_DDR3}; 241 const uint32_t rw_addr[] = {CDD_CHA_RW_1_1_DDR3, 242 CDD_CHA_RW_1_0_DDR3, 243 CDD_CHA_RW_0_1_DDR3}; 244 const uint32_t wr_addr[] = {CDD_CHA_WR_1_1_DDR3, 245 CDD_CHA_WR_1_0_DDR3, 246 CDD_CHA_WR_0_1_DDR3}; 247 248 cdd_rr = get_max_cdd(rr_addr, ARRAY_SIZE(rr_addr)); 249 cdd_rw = get_max_cdd(rw_addr, ARRAY_SIZE(rw_addr)); 250 cdd_wr = get_max_cdd(wr_addr, ARRAY_SIZE(wr_addr)); 251 cdd_ww = get_max_cdd(ww_addr, ARRAY_SIZE(ww_addr)); 252 } 253 254 /* Update max CDD values if needed */ 255 if (cdd_rr > tr_res.cdd.rr) { 256 tr_res.cdd.rr = cdd_rr; 257 } 258 if (cdd_rw > tr_res.cdd.rw) { 259 tr_res.cdd.rw = cdd_rw; 260 } 261 if (cdd_wr > tr_res.cdd.wr) { 262 tr_res.cdd.wr = cdd_wr; 263 } 264 if (cdd_ww > tr_res.cdd.ww) { 265 tr_res.cdd.ww = cdd_ww; 266 } 267 } 268 } 269 270 /* Read trained VrefCA from message block and store average value */ 271 void read_vref_ca(void) 272 { 273 const uint32_t rank0_vref_addr[] = {VREF_CA_A0, VREF_CA_B0}; 274 const uint32_t rank01_vref_addr[] = {VREF_CA_A0, VREF_CA_A1, 275 VREF_CA_B0, VREF_CA_B1}; 276 uint32_t mstr; 277 278 /* Check MSTR.active_ranks to identify multi-rank configurations */ 279 mstr = mmio_read_32(DDRC_BASE); 280 if ((mstr & MSTR_ACT_RANKS_MASK) == MSTR_DUAL_RANK_VAL) { 281 tr_res.vref_ca = get_avg_vref(rank01_vref_addr, 282 ARRAY_SIZE(rank01_vref_addr)); 283 } else { 284 tr_res.vref_ca = get_avg_vref(rank0_vref_addr, 285 ARRAY_SIZE(rank0_vref_addr)); 286 } 287 } 288 289 /* Read trained VrefDQ from message block and store average value*/ 290 void read_vref_dq(void) 291 { 292 const uint32_t rank0_vref_addr[] = {VREF_DQ_A0, VREF_DQ_B0}; 293 const uint32_t rank01_vref_addr[] = {VREF_DQ_A0, VREF_DQ_A1, 294 VREF_DQ_B0, VREF_DQ_B1}; 295 uint32_t mstr; 296 297 /* Check MSTR.active_ranks to identify multi-rank configurations */ 298 mstr = mmio_read_32(DDRC_BASE); 299 if ((mstr & MSTR_ACT_RANKS_MASK) == MSTR_DUAL_RANK_VAL) { 300 tr_res.vref_dq = get_avg_vref(rank01_vref_addr, 301 ARRAY_SIZE(rank01_vref_addr)); 302 } else { 303 tr_res.vref_dq = get_avg_vref(rank0_vref_addr, 304 ARRAY_SIZE(rank0_vref_addr)); 305 } 306 } 307 308 /* Calculate DFITMG1.dfi_t_wrdata_delay */ 309 void compute_tphy_wrdata_delay(void) 310 { 311 uint16_t tx_dqsdly, tx_dqsdly_tg1, tctrl_delay, burst_length, 312 wrdata_use_dfi_phy_clk; 313 314 const uint32_t single_rank_dly_addr[] = { 315 DBYTE0_TXDQSDLYTG0_U0, DBYTE0_TXDQSDLYTG0_U1, 316 DBYTE1_TXDQSDLYTG0_U0, DBYTE1_TXDQSDLYTG0_U1, 317 DBYTE2_TXDQSDLYTG0_U0, DBYTE2_TXDQSDLYTG0_U1, 318 DBYTE3_TXDQSDLYTG0_U0, DBYTE3_TXDQSDLYTG0_U1 319 }; 320 321 const uint32_t dual_rank_dly_addr[] = { 322 DBYTE0_TXDQSDLYTG1_U0, DBYTE0_TXDQSDLYTG1_U1, 323 DBYTE1_TXDQSDLYTG1_U0, DBYTE1_TXDQSDLYTG1_U1, 324 DBYTE2_TXDQSDLYTG1_U0, DBYTE2_TXDQSDLYTG1_U1, 325 DBYTE3_TXDQSDLYTG1_U0, DBYTE3_TXDQSDLYTG1_U1 326 }; 327 328 uint32_t mstr, dfitmg0; 329 330 /* Compute max tx_dqdqsdly for rank 0 */ 331 tx_dqsdly = get_max_delay(single_rank_dly_addr, 332 ARRAY_SIZE(single_rank_dly_addr)); 333 334 /* Check MSTR.active_ranks to identify multi-rank configurations */ 335 mstr = mmio_read_32(DDRC_BASE); 336 if ((mstr & MSTR_ACT_RANKS_MASK) == MSTR_DUAL_RANK_VAL) { 337 /* Compute max tx_dqdqsdly for rank 1 */ 338 tx_dqsdly_tg1 = get_max_delay(dual_rank_dly_addr, 339 ARRAY_SIZE(dual_rank_dly_addr)); 340 if (tx_dqsdly_tg1 > tx_dqsdly) { 341 tx_dqsdly = tx_dqsdly_tg1; 342 } 343 } 344 345 /* Extract coarse delay value + 1 for fine delay */ 346 tx_dqsdly = (tx_dqsdly >> TXDQDLY_COARSE) + 1U; 347 348 /* Compute tctrl_delay */ 349 tctrl_delay = (uint16_t)((mmio_read_16(ARDPTR_INITVAL_ADDR) / 2U) + 350 (DDRPHY_PIPE_DFI_MISC * 2U) + 3U); 351 352 burst_length = (uint16_t)(mstr >> MSTR_BURST_RDWR_POS) & 353 MSTR_BURST_RDWR_MASK; 354 dfitmg0 = mmio_read_16(DDRC_BASE + OFFSET_DDRC_DFITMG0); 355 wrdata_use_dfi_phy_clk = (uint16_t)(dfitmg0 >> DFITMG0_PHY_CLK_POS) & 356 DFITMG0_PHY_CLK_MASK; 357 358 /* Program */ 359 tr_res.tphy_wrdata_delay = tctrl_delay + 6U + burst_length + 360 wrdata_use_dfi_phy_clk + tx_dqsdly; 361 tr_res.tphy_wrdata_delay = (tr_res.tphy_wrdata_delay / 2U) + 362 (tr_res.tphy_wrdata_delay % 2U); 363 } 364 365 /* Check if memory type is LPDDR4 using MSTR register */ 366 static bool is_lpddr4(void) 367 { 368 uint32_t mstr; 369 370 mstr = mmio_read_32(DDRC_BASE); 371 return ((mstr & MSTR_DRAM_MASK) == MSTR_LPDDR4_VAL); 372 } 373 374 /* 375 * Get maximum critical delay difference value. 376 * @param cdd_addr[] - list of CDD memory addresses 377 * @param size - number of CDDs to be read 378 * @return max CDD value 379 */ 380 static uint8_t get_max_cdd(const uint32_t cdd_addr[], size_t size) 381 { 382 uint8_t cdd, max = 0; 383 int8_t signed_cdd; 384 size_t i; 385 386 for (i = 0; i < size; i++) { 387 /* CDD has type int8_t - read as unsigned and cast to signed */ 388 signed_cdd = (int8_t)(mmio_read_8(cdd_addr[i])); 389 /* We need to use absolute value */ 390 cdd = (uint8_t)((signed_cdd >= 0) ? signed_cdd : -signed_cdd); 391 max = MAX(cdd, max); 392 } 393 return max; 394 } 395 396 /* 397 * Get maximum delay value. 398 * @param delay_addr[] - list of CDD memory addresses 399 * @param size - number of values to be read 400 * @return max delay value 401 */ 402 static uint16_t get_max_delay(const uint32_t delay_addr[], size_t size) 403 { 404 uint16_t value, max = 0; 405 size_t i; 406 407 for (i = 0; i < size; i++) { 408 value = mmio_read_16(delay_addr[i]); 409 max = MAX(value, max); 410 } 411 return max; 412 } 413 414 /* 415 * Compute average vref value. 416 * @param vref_addr[] - list of vref memory addresses 417 * @param size - number of values to be read 418 * @return average vref value 419 */ 420 static uint8_t get_avg_vref(const uint32_t vref_addr[], size_t size) 421 { 422 uint32_t sum = 0; 423 size_t i; 424 425 for (i = 0; i < size; i++) { 426 sum += mmio_read_8(vref_addr[i]); 427 } 428 429 assert((sum / size) <= UINT8_MAX); 430 431 return (uint8_t)(sum / size); 432 } 433