1 /* 2 * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 #include <debug.h> 7 #include <arch_helpers.h> 8 #include <platform_def.h> 9 #include <plat_private.h> 10 #include <dram.h> 11 #include <pmu_regs.h> 12 #include <rk3399_def.h> 13 #include <secure.h> 14 #include <soc.h> 15 #include <suspend.h> 16 17 #define PMUGRF_OS_REG0 0x300 18 #define PMUGRF_OS_REG1 0x304 19 #define PMUGRF_OS_REG2 0x308 20 #define PMUGRF_OS_REG3 0x30c 21 22 #define CRU_SFTRST_DDR_CTRL(ch, n) ((0x1 << (8 + 16 + (ch) * 4)) | \ 23 ((n) << (8 + (ch) * 4))) 24 #define CRU_SFTRST_DDR_PHY(ch, n) ((0x1 << (9 + 16 + (ch) * 4)) | \ 25 ((n) << (9 + (ch) * 4))) 26 27 #define FBDIV_ENC(n) ((n) << 16) 28 #define FBDIV_DEC(n) (((n) >> 16) & 0xfff) 29 #define POSTDIV2_ENC(n) ((n) << 12) 30 #define POSTDIV2_DEC(n) (((n) >> 12) & 0x7) 31 #define POSTDIV1_ENC(n) ((n) << 8) 32 #define POSTDIV1_DEC(n) (((n) >> 8) & 0x7) 33 #define REFDIV_ENC(n) (n) 34 #define REFDIV_DEC(n) ((n) & 0x3f) 35 36 /* PMU CRU */ 37 #define PMUCRU_RSTNHOLD_CON0 0x120 38 #define PMUCRU_RSTNHOLD_CON1 0x124 39 40 #define PRESET_GPIO0_HOLD(n) (((n) << 7) | WMSK_BIT(7)) 41 #define PRESET_GPIO1_HOLD(n) (((n) << 8) | WMSK_BIT(8)) 42 43 #define SYS_COUNTER_FREQ_IN_MHZ (SYS_COUNTER_FREQ_IN_TICKS / 1000000) 44 45 /* 46 * Copy @num registers from @src to @dst 47 */ 48 __sramfunc void sram_regcpy(uintptr_t dst, uintptr_t src, uint32_t num) 49 { 50 while (num--) { 51 mmio_write_32(dst, mmio_read_32(src)); 52 dst += sizeof(uint32_t); 53 src += sizeof(uint32_t); 54 } 55 } 56 57 static __sramfunc uint32_t sram_get_timer_value(void) 58 { 59 /* 60 * Generic delay timer implementation expects the timer to be a down 61 * counter. We apply bitwise NOT operator to the tick values returned 62 * by read_cntpct_el0() to simulate the down counter. 63 */ 64 return (uint32_t)(~read_cntpct_el0()); 65 } 66 67 static __sramfunc void sram_udelay(uint32_t usec) 68 { 69 uint32_t start, cnt, delta, delta_us; 70 71 /* counter is decreasing */ 72 start = sram_get_timer_value(); 73 do { 74 cnt = sram_get_timer_value(); 75 if (cnt > start) { 76 delta = UINT32_MAX - cnt; 77 delta += start; 78 } else 79 delta = start - cnt; 80 delta_us = (delta * SYS_COUNTER_FREQ_IN_MHZ); 81 } while (delta_us < usec); 82 } 83 84 static __sramfunc void configure_sgrf(void) 85 { 86 /* 87 * SGRF_DDR_RGN_DPLL_CLK and SGRF_DDR_RGN_RTC_CLK: 88 * IC ECO bug, need to set this register. 89 * 90 * SGRF_DDR_RGN_BYPS: 91 * After the PD_CENTER suspend/resume, the DDR region 92 * related registers in the SGRF will be reset, we 93 * need to re-initialize them. 94 */ 95 mmio_write_32(SGRF_BASE + SGRF_DDRRGN_CON0_16(16), 96 SGRF_DDR_RGN_DPLL_CLK | 97 SGRF_DDR_RGN_RTC_CLK | 98 SGRF_DDR_RGN_BYPS); 99 } 100 101 static __sramfunc void rkclk_ddr_reset(uint32_t channel, uint32_t ctl, 102 uint32_t phy) 103 { 104 channel &= 0x1; 105 ctl &= 0x1; 106 phy &= 0x1; 107 mmio_write_32(CRU_BASE + CRU_SOFTRST_CON(4), 108 CRU_SFTRST_DDR_CTRL(channel, ctl) | 109 CRU_SFTRST_DDR_PHY(channel, phy)); 110 } 111 112 static __sramfunc void phy_pctrl_reset(uint32_t ch) 113 { 114 rkclk_ddr_reset(ch, 1, 1); 115 sram_udelay(10); 116 rkclk_ddr_reset(ch, 1, 0); 117 sram_udelay(10); 118 rkclk_ddr_reset(ch, 0, 0); 119 sram_udelay(10); 120 } 121 122 static __sramfunc void phy_dll_bypass_set(uint32_t ch, uint32_t hz) 123 { 124 if (hz <= 125 * MHz) { 125 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */ 126 mmio_setbits_32(PHY_REG(ch, 86), (0x3 << 2) << 8); 127 mmio_setbits_32(PHY_REG(ch, 214), (0x3 << 2) << 8); 128 mmio_setbits_32(PHY_REG(ch, 342), (0x3 << 2) << 8); 129 mmio_setbits_32(PHY_REG(ch, 470), (0x3 << 2) << 8); 130 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */ 131 mmio_setbits_32(PHY_REG(ch, 547), (0x3 << 2) << 16); 132 mmio_setbits_32(PHY_REG(ch, 675), (0x3 << 2) << 16); 133 mmio_setbits_32(PHY_REG(ch, 803), (0x3 << 2) << 16); 134 } else { 135 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */ 136 mmio_clrbits_32(PHY_REG(ch, 86), (0x3 << 2) << 8); 137 mmio_clrbits_32(PHY_REG(ch, 214), (0x3 << 2) << 8); 138 mmio_clrbits_32(PHY_REG(ch, 342), (0x3 << 2) << 8); 139 mmio_clrbits_32(PHY_REG(ch, 470), (0x3 << 2) << 8); 140 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */ 141 mmio_clrbits_32(PHY_REG(ch, 547), (0x3 << 2) << 16); 142 mmio_clrbits_32(PHY_REG(ch, 675), (0x3 << 2) << 16); 143 mmio_clrbits_32(PHY_REG(ch, 803), (0x3 << 2) << 16); 144 } 145 } 146 147 static __sramfunc void set_cs_training_index(uint32_t ch, uint32_t rank) 148 { 149 /* PHY_8/136/264/392 phy_per_cs_training_index_X 1bit offset_24 */ 150 mmio_clrsetbits_32(PHY_REG(ch, 8), 0x1 << 24, rank << 24); 151 mmio_clrsetbits_32(PHY_REG(ch, 136), 0x1 << 24, rank << 24); 152 mmio_clrsetbits_32(PHY_REG(ch, 264), 0x1 << 24, rank << 24); 153 mmio_clrsetbits_32(PHY_REG(ch, 392), 0x1 << 24, rank << 24); 154 } 155 156 static __sramfunc void select_per_cs_training_index(uint32_t ch, uint32_t rank) 157 { 158 /* PHY_84 PHY_PER_CS_TRAINING_EN_0 1bit offset_16 */ 159 if ((mmio_read_32(PHY_REG(ch, 84)) >> 16) & 1) 160 set_cs_training_index(ch, rank); 161 } 162 163 static void override_write_leveling_value(uint32_t ch) 164 { 165 uint32_t byte; 166 167 /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */ 168 mmio_setbits_32(PHY_REG(ch, 896), 1); 169 170 /* 171 * PHY_8/136/264/392 172 * phy_per_cs_training_multicast_en_X 1bit offset_16 173 */ 174 mmio_clrsetbits_32(PHY_REG(ch, 8), 0x1 << 16, 1 << 16); 175 mmio_clrsetbits_32(PHY_REG(ch, 136), 0x1 << 16, 1 << 16); 176 mmio_clrsetbits_32(PHY_REG(ch, 264), 0x1 << 16, 1 << 16); 177 mmio_clrsetbits_32(PHY_REG(ch, 392), 0x1 << 16, 1 << 16); 178 179 for (byte = 0; byte < 4; byte++) 180 mmio_clrsetbits_32(PHY_REG(ch, 63 + (128 * byte)), 181 0xffff << 16, 182 0x200 << 16); 183 184 /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */ 185 mmio_clrbits_32(PHY_REG(ch, 896), 1); 186 187 /* CTL_200 ctrlupd_req 1bit offset_8 */ 188 mmio_clrsetbits_32(CTL_REG(ch, 200), 0x1 << 8, 0x1 << 8); 189 } 190 191 static __sramfunc int data_training(uint32_t ch, 192 struct rk3399_sdram_params *sdram_params, 193 uint32_t training_flag) 194 { 195 uint32_t obs_0, obs_1, obs_2, obs_3, obs_err = 0; 196 uint32_t rank = sdram_params->ch[ch].rank; 197 uint32_t rank_mask; 198 uint32_t i, tmp; 199 200 if (sdram_params->dramtype == LPDDR4) 201 rank_mask = (rank == 1) ? 0x5 : 0xf; 202 else 203 rank_mask = (rank == 1) ? 0x1 : 0x3; 204 205 /* PHY_927 PHY_PAD_DQS_DRIVE RPULL offset_22 */ 206 mmio_setbits_32(PHY_REG(ch, 927), (1 << 22)); 207 208 if (training_flag == PI_FULL_TRAINING) { 209 if (sdram_params->dramtype == LPDDR4) { 210 training_flag = PI_WRITE_LEVELING | 211 PI_READ_GATE_TRAINING | 212 PI_READ_LEVELING | 213 PI_WDQ_LEVELING; 214 } else if (sdram_params->dramtype == LPDDR3) { 215 training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING | 216 PI_READ_GATE_TRAINING; 217 } else if (sdram_params->dramtype == DDR3) { 218 training_flag = PI_WRITE_LEVELING | 219 PI_READ_GATE_TRAINING | 220 PI_READ_LEVELING; 221 } 222 } 223 224 /* ca training(LPDDR4,LPDDR3 support) */ 225 if ((training_flag & PI_CA_TRAINING) == PI_CA_TRAINING) { 226 for (i = 0; i < 4; i++) { 227 if (!(rank_mask & (1 << i))) 228 continue; 229 230 select_per_cs_training_index(ch, i); 231 /* PI_100 PI_CALVL_EN:RW:8:2 */ 232 mmio_clrsetbits_32(PI_REG(ch, 100), 0x3 << 8, 0x2 << 8); 233 234 /* PI_92 PI_CALVL_REQ:WR:16:1,PI_CALVL_CS:RW:24:2 */ 235 mmio_clrsetbits_32(PI_REG(ch, 92), 236 (0x1 << 16) | (0x3 << 24), 237 (0x1 << 16) | (i << 24)); 238 while (1) { 239 /* PI_174 PI_INT_STATUS:RD:8:18 */ 240 tmp = mmio_read_32(PI_REG(ch, 174)) >> 8; 241 242 /* 243 * check status obs 244 * PHY_532/660/788 phy_adr_calvl_obs1_:0:32 245 */ 246 obs_0 = mmio_read_32(PHY_REG(ch, 532)); 247 obs_1 = mmio_read_32(PHY_REG(ch, 660)); 248 obs_2 = mmio_read_32(PHY_REG(ch, 788)); 249 if (((obs_0 >> 30) & 0x3) || 250 ((obs_1 >> 30) & 0x3) || 251 ((obs_2 >> 30) & 0x3)) 252 obs_err = 1; 253 if ((((tmp >> 11) & 0x1) == 0x1) && 254 (((tmp >> 13) & 0x1) == 0x1) && 255 (((tmp >> 5) & 0x1) == 0x0) && 256 (obs_err == 0)) 257 break; 258 else if ((((tmp >> 5) & 0x1) == 0x1) || 259 (obs_err == 1)) 260 return -1; 261 } 262 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 263 mmio_write_32(PI_REG(ch, 175), 0x00003f7c); 264 } 265 mmio_clrbits_32(PI_REG(ch, 100), 0x3 << 8); 266 } 267 268 /* write leveling(LPDDR4,LPDDR3,DDR3 support) */ 269 if ((training_flag & PI_WRITE_LEVELING) == PI_WRITE_LEVELING) { 270 for (i = 0; i < rank; i++) { 271 select_per_cs_training_index(ch, i); 272 /* PI_60 PI_WRLVL_EN:RW:8:2 */ 273 mmio_clrsetbits_32(PI_REG(ch, 60), 0x3 << 8, 0x2 << 8); 274 /* PI_59 PI_WRLVL_REQ:WR:8:1,PI_WRLVL_CS:RW:16:2 */ 275 mmio_clrsetbits_32(PI_REG(ch, 59), 276 (0x1 << 8) | (0x3 << 16), 277 (0x1 << 8) | (i << 16)); 278 279 while (1) { 280 /* PI_174 PI_INT_STATUS:RD:8:18 */ 281 tmp = mmio_read_32(PI_REG(ch, 174)) >> 8; 282 283 /* 284 * check status obs, if error maybe can not 285 * get leveling done PHY_40/168/296/424 286 * phy_wrlvl_status_obs_X:0:13 287 */ 288 obs_0 = mmio_read_32(PHY_REG(ch, 40)); 289 obs_1 = mmio_read_32(PHY_REG(ch, 168)); 290 obs_2 = mmio_read_32(PHY_REG(ch, 296)); 291 obs_3 = mmio_read_32(PHY_REG(ch, 424)); 292 if (((obs_0 >> 12) & 0x1) || 293 ((obs_1 >> 12) & 0x1) || 294 ((obs_2 >> 12) & 0x1) || 295 ((obs_3 >> 12) & 0x1)) 296 obs_err = 1; 297 if ((((tmp >> 10) & 0x1) == 0x1) && 298 (((tmp >> 13) & 0x1) == 0x1) && 299 (((tmp >> 4) & 0x1) == 0x0) && 300 (obs_err == 0)) 301 break; 302 else if ((((tmp >> 4) & 0x1) == 0x1) || 303 (obs_err == 1)) 304 return -1; 305 } 306 307 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 308 mmio_write_32(PI_REG(ch, 175), 0x00003f7c); 309 } 310 override_write_leveling_value(ch); 311 mmio_clrbits_32(PI_REG(ch, 60), 0x3 << 8); 312 } 313 314 /* read gate training(LPDDR4,LPDDR3,DDR3 support) */ 315 if ((training_flag & PI_READ_GATE_TRAINING) == PI_READ_GATE_TRAINING) { 316 for (i = 0; i < rank; i++) { 317 select_per_cs_training_index(ch, i); 318 /* PI_80 PI_RDLVL_GATE_EN:RW:24:2 */ 319 mmio_clrsetbits_32(PI_REG(ch, 80), 0x3 << 24, 320 0x2 << 24); 321 /* 322 * PI_74 PI_RDLVL_GATE_REQ:WR:16:1 323 * PI_RDLVL_CS:RW:24:2 324 */ 325 mmio_clrsetbits_32(PI_REG(ch, 74), 326 (0x1 << 16) | (0x3 << 24), 327 (0x1 << 16) | (i << 24)); 328 329 while (1) { 330 /* PI_174 PI_INT_STATUS:RD:8:18 */ 331 tmp = mmio_read_32(PI_REG(ch, 174)) >> 8; 332 333 /* 334 * check status obs 335 * PHY_43/171/299/427 336 * PHY_GTLVL_STATUS_OBS_x:16:8 337 */ 338 obs_0 = mmio_read_32(PHY_REG(ch, 43)); 339 obs_1 = mmio_read_32(PHY_REG(ch, 171)); 340 obs_2 = mmio_read_32(PHY_REG(ch, 299)); 341 obs_3 = mmio_read_32(PHY_REG(ch, 427)); 342 if (((obs_0 >> (16 + 6)) & 0x3) || 343 ((obs_1 >> (16 + 6)) & 0x3) || 344 ((obs_2 >> (16 + 6)) & 0x3) || 345 ((obs_3 >> (16 + 6)) & 0x3)) 346 obs_err = 1; 347 if ((((tmp >> 9) & 0x1) == 0x1) && 348 (((tmp >> 13) & 0x1) == 0x1) && 349 (((tmp >> 3) & 0x1) == 0x0) && 350 (obs_err == 0)) 351 break; 352 else if ((((tmp >> 3) & 0x1) == 0x1) || 353 (obs_err == 1)) 354 return -1; 355 } 356 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 357 mmio_write_32(PI_REG(ch, 175), 0x00003f7c); 358 } 359 mmio_clrbits_32(PI_REG(ch, 80), 0x3 << 24); 360 } 361 362 /* read leveling(LPDDR4,LPDDR3,DDR3 support) */ 363 if ((training_flag & PI_READ_LEVELING) == PI_READ_LEVELING) { 364 for (i = 0; i < rank; i++) { 365 select_per_cs_training_index(ch, i); 366 /* PI_80 PI_RDLVL_EN:RW:16:2 */ 367 mmio_clrsetbits_32(PI_REG(ch, 80), 0x3 << 16, 368 0x2 << 16); 369 /* PI_74 PI_RDLVL_REQ:WR:8:1,PI_RDLVL_CS:RW:24:2 */ 370 mmio_clrsetbits_32(PI_REG(ch, 74), 371 (0x1 << 8) | (0x3 << 24), 372 (0x1 << 8) | (i << 24)); 373 while (1) { 374 /* PI_174 PI_INT_STATUS:RD:8:18 */ 375 tmp = mmio_read_32(PI_REG(ch, 174)) >> 8; 376 377 /* 378 * make sure status obs not report error bit 379 * PHY_46/174/302/430 380 * phy_rdlvl_status_obs_X:16:8 381 */ 382 if ((((tmp >> 8) & 0x1) == 0x1) && 383 (((tmp >> 13) & 0x1) == 0x1) && 384 (((tmp >> 2) & 0x1) == 0x0)) 385 break; 386 else if (((tmp >> 2) & 0x1) == 0x1) 387 return -1; 388 } 389 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 390 mmio_write_32(PI_REG(ch, 175), 0x00003f7c); 391 } 392 mmio_clrbits_32(PI_REG(ch, 80), 0x3 << 16); 393 } 394 395 /* wdq leveling(LPDDR4 support) */ 396 if ((training_flag & PI_WDQ_LEVELING) == PI_WDQ_LEVELING) { 397 for (i = 0; i < 4; i++) { 398 if (!(rank_mask & (1 << i))) 399 continue; 400 401 select_per_cs_training_index(ch, i); 402 /* 403 * disable PI_WDQLVL_VREF_EN before wdq leveling? 404 * PI_181 PI_WDQLVL_VREF_EN:RW:8:1 405 */ 406 mmio_clrbits_32(PI_REG(ch, 181), 0x1 << 8); 407 /* PI_124 PI_WDQLVL_EN:RW:16:2 */ 408 mmio_clrsetbits_32(PI_REG(ch, 124), 0x3 << 16, 409 0x2 << 16); 410 /* PI_121 PI_WDQLVL_REQ:WR:8:1,PI_WDQLVL_CS:RW:16:2 */ 411 mmio_clrsetbits_32(PI_REG(ch, 121), 412 (0x1 << 8) | (0x3 << 16), 413 (0x1 << 8) | (i << 16)); 414 while (1) { 415 /* PI_174 PI_INT_STATUS:RD:8:18 */ 416 tmp = mmio_read_32(PI_REG(ch, 174)) >> 8; 417 if ((((tmp >> 12) & 0x1) == 0x1) && 418 (((tmp >> 13) & 0x1) == 0x1) && 419 (((tmp >> 6) & 0x1) == 0x0)) 420 break; 421 else if (((tmp >> 6) & 0x1) == 0x1) 422 return -1; 423 } 424 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 425 mmio_write_32(PI_REG(ch, 175), 0x00003f7c); 426 } 427 mmio_clrbits_32(PI_REG(ch, 124), 0x3 << 16); 428 } 429 430 /* PHY_927 PHY_PAD_DQS_DRIVE RPULL offset_22 */ 431 mmio_clrbits_32(PHY_REG(ch, 927), (1 << 22)); 432 433 return 0; 434 } 435 436 static __sramfunc void set_ddrconfig(struct rk3399_sdram_params *sdram_params, 437 unsigned char channel, uint32_t ddrconfig) 438 { 439 /* only need to set ddrconfig */ 440 struct rk3399_sdram_channel *ch = &sdram_params->ch[channel]; 441 unsigned int cs0_cap = 0; 442 unsigned int cs1_cap = 0; 443 444 cs0_cap = (1 << (ch->cs0_row + ch->col + ch->bk + ch->bw - 20)); 445 if (ch->rank > 1) 446 cs1_cap = cs0_cap >> (ch->cs0_row - ch->cs1_row); 447 if (ch->row_3_4) { 448 cs0_cap = cs0_cap * 3 / 4; 449 cs1_cap = cs1_cap * 3 / 4; 450 } 451 452 mmio_write_32(MSCH_BASE(channel) + MSCH_DEVICECONF, 453 ddrconfig | (ddrconfig << 6)); 454 mmio_write_32(MSCH_BASE(channel) + MSCH_DEVICESIZE, 455 ((cs0_cap / 32) & 0xff) | (((cs1_cap / 32) & 0xff) << 8)); 456 } 457 458 static __sramfunc void dram_all_config(struct rk3399_sdram_params *sdram_params) 459 { 460 unsigned int i; 461 462 for (i = 0; i < 2; i++) { 463 struct rk3399_sdram_channel *info = &sdram_params->ch[i]; 464 struct rk3399_msch_timings *noc = &info->noc_timings; 465 466 if (sdram_params->ch[i].col == 0) 467 continue; 468 469 mmio_write_32(MSCH_BASE(i) + MSCH_DDRTIMINGA0, 470 noc->ddrtiminga0.d32); 471 mmio_write_32(MSCH_BASE(i) + MSCH_DDRTIMINGB0, 472 noc->ddrtimingb0.d32); 473 mmio_write_32(MSCH_BASE(i) + MSCH_DDRTIMINGC0, 474 noc->ddrtimingc0.d32); 475 mmio_write_32(MSCH_BASE(i) + MSCH_DEVTODEV0, 476 noc->devtodev0.d32); 477 mmio_write_32(MSCH_BASE(i) + MSCH_DDRMODE, noc->ddrmode.d32); 478 479 /* rank 1 memory clock disable (dfi_dram_clk_disable = 1) */ 480 if (sdram_params->ch[i].rank == 1) 481 mmio_setbits_32(CTL_REG(i, 276), 1 << 17); 482 } 483 484 DDR_STRIDE(sdram_params->stride); 485 486 /* reboot hold register set */ 487 mmio_write_32(PMUCRU_BASE + CRU_PMU_RSTHOLD_CON(1), 488 CRU_PMU_SGRF_RST_RLS | 489 PRESET_GPIO0_HOLD(1) | 490 PRESET_GPIO1_HOLD(1)); 491 mmio_clrsetbits_32(CRU_BASE + CRU_GLB_RST_CON, 0x3, 0x3); 492 } 493 494 static __sramfunc void pctl_cfg(uint32_t ch, 495 struct rk3399_sdram_params *sdram_params) 496 { 497 const uint32_t *params_ctl = sdram_params->pctl_regs.denali_ctl; 498 const uint32_t *params_phy = sdram_params->phy_regs.denali_phy; 499 const uint32_t *params_pi = sdram_params->pi_regs.denali_pi; 500 uint32_t tmp, tmp1, tmp2; 501 502 /* 503 * Workaround controller bug: 504 * Do not program DRAM_CLASS until NO_PHY_IND_TRAIN_INT is programmed 505 */ 506 sram_regcpy(CTL_REG(ch, 1), (uintptr_t)¶ms_ctl[1], 507 CTL_REG_NUM - 1); 508 mmio_write_32(CTL_REG(ch, 0), params_ctl[0]); 509 sram_regcpy(PI_REG(ch, 0), (uintptr_t)¶ms_pi[0], 510 PI_REG_NUM); 511 512 mmio_write_32(PHY_REG(ch, 910), params_phy[910]); 513 mmio_write_32(PHY_REG(ch, 911), params_phy[911]); 514 mmio_write_32(PHY_REG(ch, 912), params_phy[912]); 515 516 mmio_clrsetbits_32(CTL_REG(ch, 68), PWRUP_SREFRESH_EXIT, 517 PWRUP_SREFRESH_EXIT); 518 519 /* PHY_DLL_RST_EN */ 520 mmio_clrsetbits_32(PHY_REG(ch, 957), 0x3 << 24, 1 << 24); 521 dmbst(); 522 523 mmio_setbits_32(PI_REG(ch, 0), START); 524 mmio_setbits_32(CTL_REG(ch, 0), START); 525 526 /* wait lock */ 527 while (1) { 528 tmp = mmio_read_32(PHY_REG(ch, 920)); 529 tmp1 = mmio_read_32(PHY_REG(ch, 921)); 530 tmp2 = mmio_read_32(PHY_REG(ch, 922)); 531 if ((((tmp >> 16) & 0x1) == 0x1) && 532 (((tmp1 >> 16) & 0x1) == 0x1) && 533 (((tmp1 >> 0) & 0x1) == 0x1) && 534 (((tmp2 >> 0) & 0x1) == 0x1)) 535 break; 536 /* if PLL bypass,don't need wait lock */ 537 if (mmio_read_32(PHY_REG(ch, 911)) & 0x1) 538 break; 539 } 540 541 sram_regcpy(PHY_REG(ch, 896), (uintptr_t)¶ms_phy[896], 63); 542 sram_regcpy(PHY_REG(ch, 0), (uintptr_t)¶ms_phy[0], 91); 543 sram_regcpy(PHY_REG(ch, 128), (uintptr_t)¶ms_phy[128], 91); 544 sram_regcpy(PHY_REG(ch, 256), (uintptr_t)¶ms_phy[256], 91); 545 sram_regcpy(PHY_REG(ch, 384), (uintptr_t)¶ms_phy[384], 91); 546 sram_regcpy(PHY_REG(ch, 512), (uintptr_t)¶ms_phy[512], 38); 547 sram_regcpy(PHY_REG(ch, 640), (uintptr_t)¶ms_phy[640], 38); 548 sram_regcpy(PHY_REG(ch, 768), (uintptr_t)¶ms_phy[768], 38); 549 } 550 551 static __sramfunc int dram_switch_to_next_index( 552 struct rk3399_sdram_params *sdram_params) 553 { 554 uint32_t ch, ch_count; 555 uint32_t fn = ((mmio_read_32(CTL_REG(0, 111)) >> 16) + 1) & 0x1; 556 557 mmio_write_32(CIC_BASE + CIC_CTRL0, 558 (((0x3 << 4) | (1 << 2) | 1) << 16) | 559 (fn << 4) | (1 << 2) | 1); 560 while (!(mmio_read_32(CIC_BASE + CIC_STATUS0) & (1 << 2))) 561 ; 562 563 mmio_write_32(CIC_BASE + CIC_CTRL0, 0x20002); 564 while (!(mmio_read_32(CIC_BASE + CIC_STATUS0) & (1 << 0))) 565 ; 566 567 ch_count = sdram_params->num_channels; 568 569 /* LPDDR4 f2 cann't do training, all training will fail */ 570 for (ch = 0; ch < ch_count; ch++) { 571 mmio_clrsetbits_32(PHY_REG(ch, 896), (0x3 << 8) | 1, 572 fn << 8); 573 574 /* data_training failed */ 575 if (data_training(ch, sdram_params, PI_FULL_TRAINING)) 576 return -1; 577 } 578 579 return 0; 580 } 581 582 /* 583 * Needs to be done for both channels at once in case of a shared reset signal 584 * between channels. 585 */ 586 static __sramfunc int pctl_start(uint32_t channel_mask, 587 struct rk3399_sdram_params *sdram_params) 588 { 589 uint32_t count; 590 uint32_t byte; 591 592 mmio_setbits_32(CTL_REG(0, 68), PWRUP_SREFRESH_EXIT); 593 mmio_setbits_32(CTL_REG(1, 68), PWRUP_SREFRESH_EXIT); 594 595 /* need de-access IO retention before controller START */ 596 if (channel_mask & (1 << 0)) 597 mmio_setbits_32(PMU_BASE + PMU_PWRMODE_CON, (1 << 19)); 598 if (channel_mask & (1 << 1)) 599 mmio_setbits_32(PMU_BASE + PMU_PWRMODE_CON, (1 << 23)); 600 601 /* PHY_DLL_RST_EN */ 602 if (channel_mask & (1 << 0)) 603 mmio_clrsetbits_32(PHY_REG(0, 957), 0x3 << 24, 604 0x2 << 24); 605 if (channel_mask & (1 << 1)) 606 mmio_clrsetbits_32(PHY_REG(1, 957), 0x3 << 24, 607 0x2 << 24); 608 609 /* check ERROR bit */ 610 if (channel_mask & (1 << 0)) { 611 count = 0; 612 while (!(mmio_read_32(CTL_REG(0, 203)) & (1 << 3))) { 613 /* CKE is low, loop 10ms */ 614 if (count > 100) 615 return -1; 616 617 sram_udelay(100); 618 count++; 619 } 620 621 mmio_clrbits_32(CTL_REG(0, 68), PWRUP_SREFRESH_EXIT); 622 623 /* Restore the PHY_RX_CAL_DQS value */ 624 for (byte = 0; byte < 4; byte++) 625 mmio_clrsetbits_32(PHY_REG(0, 57 + 128 * byte), 626 0xfff << 16, 627 sdram_params->rx_cal_dqs[0][byte]); 628 } 629 if (channel_mask & (1 << 1)) { 630 count = 0; 631 while (!(mmio_read_32(CTL_REG(1, 203)) & (1 << 3))) { 632 /* CKE is low, loop 10ms */ 633 if (count > 100) 634 return -1; 635 636 sram_udelay(100); 637 count++; 638 } 639 640 mmio_clrbits_32(CTL_REG(1, 68), PWRUP_SREFRESH_EXIT); 641 642 /* Restore the PHY_RX_CAL_DQS value */ 643 for (byte = 0; byte < 4; byte++) 644 mmio_clrsetbits_32(PHY_REG(1, 57 + 128 * byte), 645 0xfff << 16, 646 sdram_params->rx_cal_dqs[1][byte]); 647 } 648 649 return 0; 650 } 651 652 void dmc_save(void) 653 { 654 struct rk3399_sdram_params *sdram_params = &sdram_config; 655 uint32_t *params_ctl; 656 uint32_t *params_pi; 657 uint32_t *params_phy; 658 uint32_t refdiv, postdiv2, postdiv1, fbdiv; 659 uint32_t tmp, ch, byte; 660 661 params_ctl = sdram_params->pctl_regs.denali_ctl; 662 params_pi = sdram_params->pi_regs.denali_pi; 663 params_phy = sdram_params->phy_regs.denali_phy; 664 665 fbdiv = mmio_read_32(CRU_BASE + CRU_PLL_CON(DPLL_ID, 0)) & 0xfff; 666 tmp = mmio_read_32(CRU_BASE + CRU_PLL_CON(DPLL_ID, 1)); 667 postdiv2 = POSTDIV2_DEC(tmp); 668 postdiv1 = POSTDIV1_DEC(tmp); 669 refdiv = REFDIV_DEC(tmp); 670 671 sdram_params->ddr_freq = ((fbdiv * 24) / 672 (refdiv * postdiv1 * postdiv2)) * MHz; 673 674 INFO("sdram_params->ddr_freq = %d\n", sdram_params->ddr_freq); 675 sdram_params->odt = (((mmio_read_32(PHY_REG(0, 5)) >> 16) & 676 0x7) != 0) ? 1 : 0; 677 678 /* copy the registers CTL PI and PHY */ 679 sram_regcpy((uintptr_t)¶ms_ctl[0], CTL_REG(0, 0), CTL_REG_NUM); 680 681 /* mask DENALI_CTL_00_DATA.START, only copy here, will trigger later */ 682 params_ctl[0] &= ~(0x1 << 0); 683 684 sram_regcpy((uintptr_t)¶ms_pi[0], PI_REG(0, 0), 685 PI_REG_NUM); 686 687 /* mask DENALI_PI_00_DATA.START, only copy here, will trigger later*/ 688 params_pi[0] &= ~(0x1 << 0); 689 690 sram_regcpy((uintptr_t)¶ms_phy[0], PHY_REG(0, 0), 91); 691 sram_regcpy((uintptr_t)¶ms_phy[128], PHY_REG(0, 128), 91); 692 sram_regcpy((uintptr_t)¶ms_phy[256], PHY_REG(0, 256), 91); 693 sram_regcpy((uintptr_t)¶ms_phy[384], PHY_REG(0, 384), 91); 694 sram_regcpy((uintptr_t)¶ms_phy[512], PHY_REG(0, 512), 38); 695 sram_regcpy((uintptr_t)¶ms_phy[640], PHY_REG(0, 640), 38); 696 sram_regcpy((uintptr_t)¶ms_phy[768], PHY_REG(0, 768), 38); 697 sram_regcpy((uintptr_t)¶ms_phy[896], PHY_REG(0, 896), 63); 698 699 for (ch = 0; ch < sdram_params->num_channels; ch++) { 700 for (byte = 0; byte < 4; byte++) 701 sdram_params->rx_cal_dqs[ch][byte] = (0xfff << 16) & 702 mmio_read_32(PHY_REG(ch, 57 + byte * 128)); 703 } 704 705 /* set DENALI_PHY_957_DATA.PHY_DLL_RST_EN = 0x1 */ 706 params_phy[957] &= ~(0x3 << 24); 707 params_phy[957] |= 1 << 24; 708 params_phy[896] |= 1; 709 params_phy[896] &= ~(0x3 << 8); 710 } 711 712 __sramfunc void dmc_restore(void) 713 { 714 struct rk3399_sdram_params *sdram_params = &sdram_config; 715 uint32_t channel_mask = 0; 716 uint32_t channel; 717 718 configure_sgrf(); 719 720 retry: 721 for (channel = 0; channel < sdram_params->num_channels; channel++) { 722 phy_pctrl_reset(channel); 723 phy_dll_bypass_set(channel, sdram_params->ddr_freq); 724 if (channel >= sdram_params->num_channels) 725 continue; 726 727 pctl_cfg(channel, sdram_params); 728 } 729 730 for (channel = 0; channel < 2; channel++) { 731 if (sdram_params->ch[channel].col) 732 channel_mask |= 1 << channel; 733 } 734 735 if (pctl_start(channel_mask, sdram_params) < 0) 736 goto retry; 737 738 for (channel = 0; channel < sdram_params->num_channels; channel++) { 739 /* LPDDR2/LPDDR3 need to wait DAI complete, max 10us */ 740 if (sdram_params->dramtype == LPDDR3) 741 sram_udelay(10); 742 743 /* If traning fail, retry to do it again. */ 744 if (data_training(channel, sdram_params, PI_FULL_TRAINING)) 745 goto retry; 746 747 set_ddrconfig(sdram_params, channel, 748 sdram_params->ch[channel].ddrconfig); 749 } 750 751 dram_all_config(sdram_params); 752 753 /* Switch to index 1 and prepare for DDR frequency switch. */ 754 dram_switch_to_next_index(sdram_params); 755 } 756