1 /* 2 * (C) Copyright 2016-2017 Rockchip Inc. 3 * 4 * SPDX-License-Identifier: GPL-2.0 5 * 6 * Adapted from coreboot. 7 */ 8 9 #include <common.h> 10 #include <clk.h> 11 #include <dm.h> 12 #include <dt-structs.h> 13 #include <ram.h> 14 #include <regmap.h> 15 #include <syscon.h> 16 #include <asm/io.h> 17 #include <asm/arch/clock.h> 18 #include <asm/arch/sdram_common.h> 19 #include <asm/arch/sdram_rk3399.h> 20 #include <asm/arch/cru_rk3399.h> 21 #include <asm/arch/grf_rk3399.h> 22 #include <asm/arch/pmu_rk3399.h> 23 #include <asm/arch/hardware.h> 24 #include <linux/err.h> 25 #include <time.h> 26 27 DECLARE_GLOBAL_DATA_PTR; 28 29 struct chan_info { 30 struct rk3399_ddr_pctl_regs *pctl; 31 struct rk3399_ddr_pi_regs *pi; 32 struct rk3399_ddr_publ_regs *publ; 33 struct msch_regs *msch; 34 }; 35 36 struct dram_info { 37 #ifdef CONFIG_TPL_BUILD 38 struct chan_info chan[2]; 39 struct clk ddr_clk; 40 struct rk3399_cru *cru; 41 struct rk3399_grf_regs *grf; 42 struct rk3399_pmu_regs *pmu; 43 struct rk3399_pmucru *pmucru; 44 struct rk3399_pmusgrf_regs *pmusgrf; 45 struct rk3399_ddr_cic_regs *cic; 46 #endif 47 struct ram_info info; 48 struct rk3399_pmugrf_regs *pmugrf; 49 }; 50 51 #define PRESET_SGRF_HOLD(n) ((0x1 << (6 + 16)) | ((n) << 6)) 52 #define PRESET_GPIO0_HOLD(n) ((0x1 << (7 + 16)) | ((n) << 7)) 53 #define PRESET_GPIO1_HOLD(n) ((0x1 << (8 + 16)) | ((n) << 8)) 54 55 #define PHY_DRV_ODT_HI_Z 0x0 56 #define PHY_DRV_ODT_240 0x1 57 #define PHY_DRV_ODT_120 0x8 58 #define PHY_DRV_ODT_80 0x9 59 #define PHY_DRV_ODT_60 0xc 60 #define PHY_DRV_ODT_48 0xd 61 #define PHY_DRV_ODT_40 0xe 62 #define PHY_DRV_ODT_34_3 0xf 63 64 #ifdef CONFIG_TPL_BUILD 65 66 struct rockchip_dmc_plat { 67 #if CONFIG_IS_ENABLED(OF_PLATDATA) 68 struct dtd_rockchip_rk3399_dmc dtplat; 69 #else 70 struct rk3399_sdram_params sdram_params; 71 #endif 72 struct regmap *map; 73 }; 74 75 u32 g_pwrup_srefresh_exit[2]; 76 77 struct rk3399_sdram_params dfs_configs[] = { 78 #include "sdram-rk3399-lpddr4-400.inc" 79 #include "sdram-rk3399-lpddr4-800.inc" 80 }; 81 82 #define CRU_SFTRST_DDR_CTRL(ch, n) ((0x1 << (8 + 16 + (ch) * 4)) | \ 83 ((n) << (8 + (ch) * 4))) 84 #define CRU_SFTRST_DDR_PHY(ch, n) ((0x1 << (9 + 16 + (ch) * 4)) | \ 85 ((n) << (9 + (ch) * 4))) 86 static void rkclk_ddr_reset(struct rk3399_cru *cru, u32 channel, u32 ctl, 87 u32 phy) 88 { 89 channel &= 0x1; 90 ctl &= 0x1; 91 phy &= 0x1; 92 writel(CRU_SFTRST_DDR_CTRL(channel, ctl) | 93 CRU_SFTRST_DDR_PHY(channel, phy), 94 &cru->softrst_con[4]); 95 } 96 97 static void phy_pctrl_reset(struct rk3399_cru *cru, 98 u32 channel) 99 { 100 rkclk_ddr_reset(cru, channel, 1, 1); 101 udelay(10); 102 rkclk_ddr_reset(cru, channel, 1, 0); 103 udelay(10); 104 rkclk_ddr_reset(cru, channel, 0, 0); 105 udelay(10); 106 } 107 108 static void phy_dll_bypass_set(struct rk3399_ddr_publ_regs *ddr_publ_regs, 109 u32 freq) 110 { 111 u32 *denali_phy = ddr_publ_regs->denali_phy; 112 113 /* From IP spec, only freq small than 125 can enter dll bypass mode */ 114 if (freq <= 125) { 115 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */ 116 setbits_le32(&denali_phy[86], (0x3 << 2) << 8); 117 setbits_le32(&denali_phy[214], (0x3 << 2) << 8); 118 setbits_le32(&denali_phy[342], (0x3 << 2) << 8); 119 setbits_le32(&denali_phy[470], (0x3 << 2) << 8); 120 121 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */ 122 setbits_le32(&denali_phy[547], (0x3 << 2) << 16); 123 setbits_le32(&denali_phy[675], (0x3 << 2) << 16); 124 setbits_le32(&denali_phy[803], (0x3 << 2) << 16); 125 } else { 126 /* phy_sw_master_mode_X PHY_86/214/342/470 4bits offset_8 */ 127 clrbits_le32(&denali_phy[86], (0x3 << 2) << 8); 128 clrbits_le32(&denali_phy[214], (0x3 << 2) << 8); 129 clrbits_le32(&denali_phy[342], (0x3 << 2) << 8); 130 clrbits_le32(&denali_phy[470], (0x3 << 2) << 8); 131 132 /* phy_adrctl_sw_master_mode PHY_547/675/803 4bits offset_16 */ 133 clrbits_le32(&denali_phy[547], (0x3 << 2) << 16); 134 clrbits_le32(&denali_phy[675], (0x3 << 2) << 16); 135 clrbits_le32(&denali_phy[803], (0x3 << 2) << 16); 136 } 137 } 138 139 static void set_memory_map(const struct chan_info *chan, u32 channel, 140 const struct rk3399_sdram_params *sdram_params) 141 { 142 const struct rk3399_sdram_channel *sdram_ch = 143 &sdram_params->ch[channel]; 144 u32 *denali_ctl = chan->pctl->denali_ctl; 145 u32 *denali_pi = chan->pi->denali_pi; 146 u32 cs_map; 147 u32 reduc; 148 u32 row; 149 150 /* Get row number from ddrconfig setting */ 151 if (sdram_ch->cap_info.ddrconfig < 2 || 152 sdram_ch->cap_info.ddrconfig == 4) 153 row = 16; 154 else if (sdram_ch->cap_info.ddrconfig == 3 || 155 sdram_ch->cap_info.ddrconfig == 5) 156 row = 14; 157 else 158 row = 15; 159 160 cs_map = (sdram_ch->cap_info.rank > 1) ? 3 : 1; 161 reduc = (sdram_ch->cap_info.bw == 2) ? 0 : 1; 162 163 /* Set the dram configuration to ctrl */ 164 clrsetbits_le32(&denali_ctl[191], 0xF, (12 - sdram_ch->cap_info.col)); 165 clrsetbits_le32(&denali_ctl[190], (0x3 << 16) | (0x7 << 24), 166 ((3 - sdram_ch->cap_info.bk) << 16) | 167 ((16 - row) << 24)); 168 169 clrsetbits_le32(&denali_ctl[196], 0x3 | (1 << 16), 170 cs_map | (reduc << 16)); 171 172 /* PI_199 PI_COL_DIFF:RW:0:4 */ 173 clrsetbits_le32(&denali_pi[199], 0xF, (12 - sdram_ch->cap_info.col)); 174 175 /* PI_155 PI_ROW_DIFF:RW:24:3 PI_BANK_DIFF:RW:16:2 */ 176 clrsetbits_le32(&denali_pi[155], (0x3 << 16) | (0x7 << 24), 177 ((3 - sdram_ch->cap_info.bk) << 16) | 178 ((16 - row) << 24)); 179 if (sdram_params->base.dramtype == LPDDR4) { 180 if (cs_map == 1) 181 cs_map = 0x5; 182 else if (cs_map == 2) 183 cs_map = 0xa; 184 else 185 cs_map = 0xF; 186 } 187 188 /* PI_41 PI_CS_MAP:RW:24:4 */ 189 clrsetbits_le32(&denali_pi[41], 0xf << 24, cs_map << 24); 190 if (sdram_ch->cap_info.rank == 1 && 191 sdram_params->base.dramtype == DDR3) 192 writel(0x2EC7FFFF, &denali_pi[34]); 193 } 194 195 static void phy_io_config(const struct chan_info *chan, 196 struct rk3399_sdram_params *sdram_params, u32 rd_vref, 197 u32 b_reg, u32 channel) 198 { 199 u32 *denali_phy; 200 u32 *denali_ctl; 201 u32 vref_mode_dq = 0, vref_value_dq = 0; 202 u32 vref_mode_ac = 0, vref_value_ac = 0; 203 u32 mode_sel = 0; 204 u32 boostp, boostn; 205 u32 slewp, slewn; 206 u32 speed; 207 u32 rx_cm_input; 208 u32 reg_value; 209 u32 ds_value, odt_value; 210 211 if (b_reg) { 212 denali_phy = chan->publ->denali_phy; 213 denali_ctl = chan->pctl->denali_ctl; 214 } else { 215 denali_phy = sdram_params->phy_regs.denali_phy; 216 denali_ctl = sdram_params->pctl_regs.denali_ctl; 217 } 218 rd_vref *= 1000; 219 /* vref setting & mode setting */ 220 if (sdram_params->base.dramtype == LPDDR4) { 221 if (rd_vref < 36700) { 222 /* MODE_LV[2:0] = LPDDR4 (Range 2)*/ 223 vref_mode_dq = 0x7; 224 /* MODE[2:0]= LPDDR4 Range 2(0.4*VDDQ) */ 225 mode_sel = 0x5; 226 vref_value_dq = (rd_vref - 3300) / 521; 227 } else { 228 /* MODE_LV[2:0] = LPDDR4 (Range 1)*/ 229 vref_mode_dq = 0x6; 230 /* MODE[2:0]= LPDDR4 Range 1(0.33*VDDQ) */ 231 mode_sel = 0x4; 232 vref_value_dq = (rd_vref - 15300) / 521; 233 } 234 vref_mode_ac = 0x6; 235 /* VDDQ/3/2=16.8% */ 236 vref_value_ac = 0x3; 237 } else if (sdram_params->base.dramtype == LPDDR3) { 238 /* LPDDR3 */ 239 if (sdram_params->base.odt == 1) { 240 vref_mode_dq = 0x5; /*LPDDR3 ODT*/ 241 ds_value = readl(&denali_ctl[138]) & 0xf; 242 odt_value = (readl(&denali_phy[6]) >> 4) & 0xf; 243 if (ds_value == 0x3) { /* 48ohm */ 244 switch (odt_value) { 245 case PHY_DRV_ODT_240: 246 vref_value_dq = 0x1B; 247 break; 248 case PHY_DRV_ODT_120: 249 vref_value_dq = 0x26; 250 break; 251 case PHY_DRV_ODT_60: 252 vref_value_dq = 0x36; 253 break; 254 } 255 } else if (ds_value == 0x2) { /* 40ohm */ 256 switch (odt_value) { 257 case PHY_DRV_ODT_240: 258 vref_value_dq = 0x19; 259 break; 260 case PHY_DRV_ODT_120: 261 vref_value_dq = 0x23; 262 break; 263 case PHY_DRV_ODT_60: 264 vref_value_dq = 0x31; 265 break; 266 } 267 } else if (ds_value == 0x1) { /*34.3ohm*/ 268 switch (odt_value) { 269 case PHY_DRV_ODT_240: 270 vref_value_dq = 0x17; 271 break; 272 case PHY_DRV_ODT_120: 273 vref_value_dq = 0x20; 274 break; 275 case PHY_DRV_ODT_60: 276 vref_value_dq = 0x2e; 277 break; 278 } 279 } 280 } else { 281 vref_mode_dq = 0x2; 282 vref_value_dq = 0x1f; 283 } 284 vref_mode_ac = 0x2; 285 vref_value_ac = 0x1f; 286 mode_sel = 0x0; 287 } else if (sdram_params->base.dramtype == DDR3) { 288 vref_mode_dq = 0x1; 289 vref_value_dq = 0x1f; 290 vref_mode_ac = 0x1; 291 vref_value_ac = 0x1f; 292 mode_sel = 0x1; 293 } 294 295 reg_value = (vref_mode_dq << 9) | (0x1 << 8) | vref_value_dq; 296 /* PHY_913 PHY_PAD_VREF_CTRL_DQ_0 12bits offset_8 */ 297 clrsetbits_le32(&denali_phy[913], 0xfff << 8, reg_value << 8); 298 /* PHY_914 PHY_PAD_VREF_CTRL_DQ_1 12bits offset_0 */ 299 clrsetbits_le32(&denali_phy[914], 0xfff, reg_value); 300 /* PHY_914 PHY_PAD_VREF_CTRL_DQ_2 12bits offset_16 */ 301 clrsetbits_le32(&denali_phy[914], 0xfff << 16, reg_value << 16); 302 /* PHY_915 PHY_PAD_VREF_CTRL_DQ_3 12bits offset_0 */ 303 clrsetbits_le32(&denali_phy[915], 0xfff, reg_value); 304 305 reg_value = (vref_mode_ac << 9) | (0x1 << 8) | vref_value_ac; 306 /* PHY_915 PHY_PAD_VREF_CTRL_AC 12bits offset_16 */ 307 clrsetbits_le32(&denali_phy[915], 0xfff << 16, reg_value << 16); 308 309 /* PHY_924 PHY_PAD_FDBK_DRIVE */ 310 clrsetbits_le32(&denali_phy[924], 0x7 << 15, mode_sel << 15); 311 /* PHY_926 PHY_PAD_DATA_DRIVE */ 312 clrsetbits_le32(&denali_phy[926], 0x7 << 6, mode_sel << 6); 313 /* PHY_927 PHY_PAD_DQS_DRIVE */ 314 clrsetbits_le32(&denali_phy[927], 0x7 << 6, mode_sel << 6); 315 /* PHY_928 PHY_PAD_ADDR_DRIVE */ 316 clrsetbits_le32(&denali_phy[928], 0x7 << 14, mode_sel << 14); 317 /* PHY_929 PHY_PAD_CLK_DRIVE */ 318 clrsetbits_le32(&denali_phy[929], 0x7 << 14, mode_sel << 14); 319 /* PHY_935 PHY_PAD_CKE_DRIVE */ 320 clrsetbits_le32(&denali_phy[935], 0x7 << 14, mode_sel << 14); 321 /* PHY_937 PHY_PAD_RST_DRIVE */ 322 clrsetbits_le32(&denali_phy[937], 0x7 << 14, mode_sel << 14); 323 /* PHY_939 PHY_PAD_CS_DRIVE */ 324 clrsetbits_le32(&denali_phy[939], 0x7 << 14, mode_sel << 14); 325 326 /* BOOSTP_EN & BOOSTn_EN */ 327 if (sdram_params->base.dramtype == LPDDR4) { 328 boostp = 0x1; 329 boostn = 0x1; 330 reg_value = ((boostp << 4) | boostn); 331 /* PHY_925 PHY_PAD_FDBK_DRIVE2 */ 332 clrsetbits_le32(&denali_phy[925], 0xff << 8, reg_value << 8); 333 /* PHY_926 PHY_PAD_DATA_DRIVE */ 334 clrsetbits_le32(&denali_phy[926], 0xff << 12, reg_value << 12); 335 /* PHY_927 PHY_PAD_DQS_DRIVE */ 336 clrsetbits_le32(&denali_phy[927], 0xff << 14, reg_value << 14); 337 /* PHY_928 PHY_PAD_ADDR_DRIVE */ 338 clrsetbits_le32(&denali_phy[928], 0xff << 20, reg_value << 20); 339 /* PHY_929 PHY_PAD_CLK_DRIVE */ 340 clrsetbits_le32(&denali_phy[929], 0xff << 22, reg_value << 22); 341 /* PHY_935 PHY_PAD_CKE_DRIVE */ 342 clrsetbits_le32(&denali_phy[935], 0xff << 20, reg_value << 20); 343 /* PHY_937 PHY_PAD_RST_DRIVE */ 344 clrsetbits_le32(&denali_phy[937], 0xff << 20, reg_value << 20); 345 /* PHY_939 PHY_PAD_CS_DRIVE */ 346 clrsetbits_le32(&denali_phy[939], 0xff << 20, reg_value << 20); 347 } 348 349 /* SLEWP & SLEWN */ 350 if (sdram_params->base.dramtype == LPDDR4) { 351 slewp = 0x1; 352 slewn = 0x1; 353 reg_value = ((slewp << 3) | slewn); 354 /* PHY_924 PHY_PAD_FDBK_DRIVE */ 355 clrsetbits_le32(&denali_phy[924], 0x3f << 8, reg_value << 8); 356 /* PHY_926 PHY_PAD_DATA_DRIVE */ 357 clrsetbits_le32(&denali_phy[926], 0x3f, reg_value); 358 /* PHY_927 PHY_PAD_DQS_DRIVE */ 359 clrsetbits_le32(&denali_phy[927], 0x3f, reg_value); 360 /* PHY_928 PHY_PAD_ADDR_DRIVE */ 361 clrsetbits_le32(&denali_phy[928], 0x3f << 8, reg_value << 8); 362 /* PHY_929 PHY_PAD_CLK_DRIVE */ 363 clrsetbits_le32(&denali_phy[929], 0x3f << 8, reg_value << 8); 364 /* PHY_935 PHY_PAD_CKE_DRIVE */ 365 clrsetbits_le32(&denali_phy[935], 0x3f << 8, reg_value << 8); 366 /* PHY_937 PHY_PAD_RST_DRIVE */ 367 clrsetbits_le32(&denali_phy[937], 0x3f << 8, reg_value << 8); 368 /* PHY_939 PHY_PAD_CS_DRIVE */ 369 clrsetbits_le32(&denali_phy[939], 0x3f << 8, reg_value << 8); 370 } 371 372 /* speed setting */ 373 if (sdram_params->base.ddr_freq < 400 * MHz) 374 speed = 0x0; 375 else if (sdram_params->base.ddr_freq < 800 * MHz) 376 speed = 0x1; 377 else if (sdram_params->base.ddr_freq < 1200 * MHz) 378 speed = 0x2; 379 speed = 0x2; 380 381 /* PHY_924 PHY_PAD_FDBK_DRIVE */ 382 clrsetbits_le32(&denali_phy[924], 0x3 << 21, speed << 21); 383 /* PHY_926 PHY_PAD_DATA_DRIVE */ 384 clrsetbits_le32(&denali_phy[926], 0x3 << 9, speed << 9); 385 /* PHY_927 PHY_PAD_DQS_DRIVE */ 386 clrsetbits_le32(&denali_phy[927], 0x3 << 9, speed << 9); 387 /* PHY_928 PHY_PAD_ADDR_DRIVE */ 388 clrsetbits_le32(&denali_phy[928], 0x3 << 17, speed << 17); 389 /* PHY_929 PHY_PAD_CLK_DRIVE */ 390 clrsetbits_le32(&denali_phy[929], 0x3 << 17, speed << 17); 391 /* PHY_935 PHY_PAD_CKE_DRIVE */ 392 clrsetbits_le32(&denali_phy[935], 0x3 << 17, speed << 17); 393 /* PHY_937 PHY_PAD_RST_DRIVE */ 394 clrsetbits_le32(&denali_phy[937], 0x3 << 17, speed << 17); 395 /* PHY_939 PHY_PAD_CS_DRIVE */ 396 clrsetbits_le32(&denali_phy[939], 0x3 << 17, speed << 17); 397 398 /* RX_CM_INPUT */ 399 if (sdram_params->base.dramtype == LPDDR4) { 400 rx_cm_input = 0x1; 401 /* PHY_924 PHY_PAD_FDBK_DRIVE */ 402 clrsetbits_le32(&denali_phy[924], 0x1 << 14, rx_cm_input << 14); 403 /* PHY_926 PHY_PAD_DATA_DRIVE */ 404 clrsetbits_le32(&denali_phy[926], 0x1 << 11, rx_cm_input << 11); 405 /* PHY_927 PHY_PAD_DQS_DRIVE */ 406 clrsetbits_le32(&denali_phy[927], 0x1 << 13, rx_cm_input << 13); 407 /* PHY_928 PHY_PAD_ADDR_DRIVE */ 408 clrsetbits_le32(&denali_phy[928], 0x1 << 19, rx_cm_input << 19); 409 /* PHY_929 PHY_PAD_CLK_DRIVE */ 410 clrsetbits_le32(&denali_phy[929], 0x1 << 21, rx_cm_input << 21); 411 /* PHY_935 PHY_PAD_CKE_DRIVE */ 412 clrsetbits_le32(&denali_phy[935], 0x1 << 19, rx_cm_input << 19); 413 /* PHY_937 PHY_PAD_RST_DRIVE */ 414 clrsetbits_le32(&denali_phy[937], 0x1 << 19, rx_cm_input << 19); 415 /* PHY_939 PHY_PAD_CS_DRIVE */ 416 clrsetbits_le32(&denali_phy[939], 0x1 << 19, rx_cm_input << 19); 417 } 418 } 419 420 struct io_setting { 421 u32 mhz; 422 u32 mr5; 423 /* dram side */ 424 u32 dq_odt; 425 u32 ca_odt; 426 u32 pdds; 427 u32 dq_vref; 428 u32 ca_vref; 429 /* phy side */ 430 u32 rd_odt; 431 u32 wr_dq_drv; 432 u32 wr_ca_drv; 433 u32 wr_ckcs_drv; 434 u32 rd_odt_en; 435 u32 rd_vref; 436 }; 437 438 struct io_setting lp4_io_set[] = { 439 { 440 50 * MHz, 441 0, 442 /* dram side */ 443 0, /* dq_odt; */ 444 0, /* ca_odt; */ 445 6, /* pdds; */ 446 0x72, /* dq_vref; */ 447 0x72, /* ca_vref; */ 448 /* phy side */ 449 PHY_DRV_ODT_HI_Z, /* rd_odt; */ 450 PHY_DRV_ODT_40, /* wr_dq_drv; */ 451 PHY_DRV_ODT_40, /* wr_ca_drv; */ 452 PHY_DRV_ODT_40, /* wr_ckcs_drv; */ 453 0, /* rd_odt_en;*/ 454 41, /* rd_vref; (unit %, range 3.3% - 48.7%) */ 455 }, 456 { 457 400 * MHz, 458 0, 459 /* dram side */ 460 0, /* dq_odt; */ 461 0, /* ca_odt; */ 462 6, /* pdds; */ 463 0x72, /* dq_vref; */ 464 0x72, /* ca_vref; */ 465 /* phy side */ 466 PHY_DRV_ODT_HI_Z, /* rd_odt; */ 467 PHY_DRV_ODT_40, /* wr_dq_drv; */ 468 PHY_DRV_ODT_40, /* wr_ca_drv; */ 469 PHY_DRV_ODT_40, /* wr_ckcs_drv; */ 470 0, /* rd_odt_en; */ 471 /* shmoo result, read signal 41% is the best */ 472 41, /* rd_vref; (unit %, range 3.3% - 48.7%) */ 473 }, 474 { 475 800 * MHz, 476 0, 477 /* dram side */ 478 0, /* dq_odt; */ 479 0, /* ca_odt; */ 480 1, /* pdds; */ 481 0x72, /* dq_vref; */ 482 0x72, /* ca_vref; */ 483 /* phy side */ 484 PHY_DRV_ODT_40, /* rd_odt; */ 485 PHY_DRV_ODT_40, /* wr_dq_drv; */ 486 PHY_DRV_ODT_40, /* wr_ca_drv; */ 487 PHY_DRV_ODT_40, /* wr_ckcs_drv; */ 488 1, /* rd_odt_en; */ 489 17, /* rd_vref; (unit %, range 3.3% - 48.7%) */ 490 }, 491 { 492 1066 * MHz, 493 0, 494 /* dram side */ 495 6, /* dq_odt; */ 496 0, /* ca_odt; */ 497 1, /* pdds; */ 498 0x10, /* dq_vref; */ 499 0x72, /* ca_vref; */ 500 /* phy side */ 501 PHY_DRV_ODT_40, /* rd_odt; */ 502 PHY_DRV_ODT_60, /* wr_dq_drv; */ 503 PHY_DRV_ODT_40, /* wr_ca_drv; */ 504 PHY_DRV_ODT_40, /* wr_ckcs_drv; */ 505 1, /* rd_odt_en; */ 506 17, /* rd_vref; (unit %, range 3.3% - 48.7%) */ 507 }, 508 }; 509 510 static struct io_setting *get_io_set(struct rk3399_sdram_params *sdram_params, 511 u32 mr5) 512 { 513 struct io_setting *io = NULL; 514 u32 n; 515 516 for (n = 0; n < ARRAY_SIZE(lp4_io_set); n++) { 517 io = &lp4_io_set[n]; 518 if (io->mr5 != 0) { 519 if (io->mhz >= sdram_params->base.ddr_freq && 520 io->mr5 == mr5) 521 break; 522 } else { 523 if (io->mhz >= sdram_params->base.ddr_freq) 524 break; 525 } 526 } 527 return io; 528 } 529 530 /* 531 * b_reg: indicate whether set phy register 532 * or just set sdram_params. 533 * if b_reg = 0, channel, mr5 are not care 534 */ 535 static void set_ds_odt(const struct chan_info *chan, 536 struct rk3399_sdram_params *sdram_params, u32 b_reg, 537 u32 channel, u32 mr5) 538 { 539 u32 *denali_phy; 540 u32 *denali_ctl; 541 u32 tsel_idle_en, tsel_wr_en, tsel_rd_en; 542 u32 tsel_idle_select_p, tsel_rd_select_p; 543 u32 tsel_idle_select_n, tsel_rd_select_n; 544 u32 tsel_wr_select_dq_p, tsel_wr_select_ca_p; 545 u32 tsel_wr_select_dq_n, tsel_wr_select_ca_n; 546 u32 soc_odt = 0; 547 u32 tsel_ckcs_select_p, tsel_ckcs_select_n; 548 u32 reg_value; 549 struct io_setting *io = NULL; 550 551 if (b_reg) { 552 denali_phy = chan->publ->denali_phy; 553 denali_ctl = chan->pctl->denali_ctl; 554 } else { 555 denali_phy = sdram_params->phy_regs.denali_phy; 556 denali_ctl = sdram_params->pctl_regs.denali_ctl; 557 } 558 559 if (sdram_params->base.dramtype == LPDDR4) { 560 io = get_io_set(sdram_params, mr5); 561 562 tsel_rd_select_p = PHY_DRV_ODT_HI_Z; 563 tsel_rd_select_n = io->rd_odt; 564 565 tsel_idle_select_p = PHY_DRV_ODT_HI_Z; 566 tsel_idle_select_n = PHY_DRV_ODT_HI_Z; 567 568 tsel_wr_select_dq_p = io->wr_dq_drv; 569 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3; 570 571 tsel_wr_select_ca_p = io->wr_ca_drv; 572 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3; 573 574 tsel_ckcs_select_p = io->wr_ckcs_drv; 575 tsel_ckcs_select_n = PHY_DRV_ODT_34_3; 576 577 switch (tsel_rd_select_n) { 578 case PHY_DRV_ODT_240: 579 soc_odt = 1; 580 break; 581 case PHY_DRV_ODT_120: 582 soc_odt = 2; 583 break; 584 case PHY_DRV_ODT_80: 585 soc_odt = 3; 586 break; 587 case PHY_DRV_ODT_60: 588 soc_odt = 4; 589 break; 590 case PHY_DRV_ODT_48: 591 soc_odt = 5; 592 break; 593 case PHY_DRV_ODT_40: 594 soc_odt = 6; 595 break; 596 case PHY_DRV_ODT_34_3: 597 soc_odt = 6; 598 printf("LPDDR4 MR22 Soc ODT not support\n"); 599 break; 600 case PHY_DRV_ODT_HI_Z: 601 default: 602 soc_odt = 0; 603 break; 604 } 605 } else if (sdram_params->base.dramtype == LPDDR3) { 606 tsel_rd_select_p = PHY_DRV_ODT_240; 607 tsel_rd_select_n = PHY_DRV_ODT_HI_Z; 608 609 tsel_idle_select_p = PHY_DRV_ODT_240; 610 tsel_idle_select_n = PHY_DRV_ODT_HI_Z; 611 612 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3; 613 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3; 614 615 tsel_wr_select_ca_p = PHY_DRV_ODT_34_3; 616 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3; 617 618 tsel_ckcs_select_p = PHY_DRV_ODT_34_3; 619 tsel_ckcs_select_n = PHY_DRV_ODT_34_3; 620 } else { 621 tsel_rd_select_p = PHY_DRV_ODT_240; 622 tsel_rd_select_n = PHY_DRV_ODT_240; 623 624 tsel_idle_select_p = PHY_DRV_ODT_240; 625 tsel_idle_select_n = PHY_DRV_ODT_240; 626 627 tsel_wr_select_dq_p = PHY_DRV_ODT_34_3; 628 tsel_wr_select_dq_n = PHY_DRV_ODT_34_3; 629 630 tsel_wr_select_ca_p = PHY_DRV_ODT_34_3; 631 tsel_wr_select_ca_n = PHY_DRV_ODT_34_3; 632 633 tsel_ckcs_select_p = PHY_DRV_ODT_34_3; 634 tsel_ckcs_select_n = PHY_DRV_ODT_34_3; 635 } 636 637 if (sdram_params->base.dramtype == LPDDR4) { 638 if (sdram_params->base.odt == 1) { 639 tsel_rd_en = io->rd_odt_en; 640 tsel_idle_en = 0; 641 } else { 642 tsel_rd_en = 0; 643 tsel_idle_en = 0; 644 } 645 } else { 646 tsel_rd_en = sdram_params->base.odt; 647 tsel_idle_en = 0; 648 } 649 tsel_wr_en = 0; 650 651 #define CS0_MR22_VAL (0) 652 #define CS1_MR22_VAL (3) 653 /* F0_0 */ 654 clrsetbits_le32(&denali_ctl[145], 0xFF << 16, 655 (soc_odt | (CS0_MR22_VAL << 3)) << 16); 656 /* F2_0, F1_0 */ 657 clrsetbits_le32(&denali_ctl[146], 0xFF00FF, 658 ((soc_odt | (CS0_MR22_VAL << 3)) << 16) | 659 (soc_odt | (CS0_MR22_VAL << 3))); 660 /* F0_1 */ 661 clrsetbits_le32(&denali_ctl[159], 0xFF << 16, 662 (soc_odt | (CS1_MR22_VAL << 3)) << 16); 663 /* F2_1, F1_1 */ 664 clrsetbits_le32(&denali_ctl[160], 0xFF00FF, 665 ((soc_odt | (CS1_MR22_VAL << 3)) << 16) | 666 (soc_odt | (CS1_MR22_VAL << 3))); 667 668 /* 669 * phy_dq_tsel_select_X 24bits DENALI_PHY_6/134/262/390 offset_0 670 * sets termination values for read/idle cycles and drive strength 671 * for write cycles for DQ/DM 672 */ 673 reg_value = tsel_rd_select_n | (tsel_rd_select_p << 0x4) | 674 (tsel_wr_select_dq_n << 8) | (tsel_wr_select_dq_p << 12) | 675 (tsel_idle_select_n << 16) | (tsel_idle_select_p << 20); 676 clrsetbits_le32(&denali_phy[6], 0xffffff, reg_value); 677 clrsetbits_le32(&denali_phy[134], 0xffffff, reg_value); 678 clrsetbits_le32(&denali_phy[262], 0xffffff, reg_value); 679 clrsetbits_le32(&denali_phy[390], 0xffffff, reg_value); 680 681 /* 682 * phy_dqs_tsel_select_X 24bits DENALI_PHY_7/135/263/391 offset_0 683 * sets termination values for read/idle cycles and drive strength 684 * for write cycles for DQS 685 */ 686 clrsetbits_le32(&denali_phy[7], 0xffffff, reg_value); 687 clrsetbits_le32(&denali_phy[135], 0xffffff, reg_value); 688 clrsetbits_le32(&denali_phy[263], 0xffffff, reg_value); 689 clrsetbits_le32(&denali_phy[391], 0xffffff, reg_value); 690 691 /* phy_adr_tsel_select_ 8bits DENALI_PHY_544/672/800 offset_0 */ 692 reg_value = tsel_wr_select_ca_n | (tsel_wr_select_ca_p << 0x4); 693 if (sdram_params->base.dramtype == LPDDR4) { 694 /* LPDDR4 these register read always return 0, so 695 * can not use clrsetbits_le32(), need to write32 696 */ 697 writel((0x300 << 8) | reg_value, &denali_phy[544]); 698 writel((0x300 << 8) | reg_value, &denali_phy[672]); 699 writel((0x300 << 8) | reg_value, &denali_phy[800]); 700 } else { 701 clrsetbits_le32(&denali_phy[544], 0xff, reg_value); 702 clrsetbits_le32(&denali_phy[672], 0xff, reg_value); 703 clrsetbits_le32(&denali_phy[800], 0xff, reg_value); 704 } 705 706 /* phy_pad_addr_drive 8bits DENALI_PHY_928 offset_0 */ 707 clrsetbits_le32(&denali_phy[928], 0xff, reg_value); 708 709 /* phy_pad_rst_drive 8bits DENALI_PHY_937 offset_0 */ 710 if (!b_reg) 711 clrsetbits_le32(&denali_phy[937], 0xff, reg_value); 712 713 /* phy_pad_cke_drive 8bits DENALI_PHY_935 offset_0 */ 714 clrsetbits_le32(&denali_phy[935], 0xff, reg_value); 715 716 /* phy_pad_cs_drive 8bits DENALI_PHY_939 offset_0 */ 717 clrsetbits_le32(&denali_phy[939], 0xff, 718 tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4)); 719 720 /* phy_pad_clk_drive 8bits DENALI_PHY_929 offset_0 */ 721 clrsetbits_le32(&denali_phy[929], 0xff, 722 tsel_ckcs_select_n | (tsel_ckcs_select_p << 0x4)); 723 724 /* phy_pad_fdbk_drive 23bit DENALI_PHY_924/925 */ 725 clrsetbits_le32(&denali_phy[924], 0xff, 726 tsel_wr_select_ca_n | (tsel_wr_select_ca_p << 4)); 727 clrsetbits_le32(&denali_phy[925], 0xff, 728 tsel_wr_select_dq_n | (tsel_wr_select_dq_p << 4)); 729 730 /* phy_dq_tsel_enable_X 3bits DENALI_PHY_5/133/261/389 offset_16 */ 731 reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2)) 732 << 16; 733 clrsetbits_le32(&denali_phy[5], 0x7 << 16, reg_value); 734 clrsetbits_le32(&denali_phy[133], 0x7 << 16, reg_value); 735 clrsetbits_le32(&denali_phy[261], 0x7 << 16, reg_value); 736 clrsetbits_le32(&denali_phy[389], 0x7 << 16, reg_value); 737 738 /* phy_dqs_tsel_enable_X 3bits DENALI_PHY_6/134/262/390 offset_24 */ 739 reg_value = (tsel_rd_en | (tsel_wr_en << 1) | (tsel_idle_en << 2)) 740 << 24; 741 clrsetbits_le32(&denali_phy[6], 0x7 << 24, reg_value); 742 clrsetbits_le32(&denali_phy[134], 0x7 << 24, reg_value); 743 clrsetbits_le32(&denali_phy[262], 0x7 << 24, reg_value); 744 clrsetbits_le32(&denali_phy[390], 0x7 << 24, reg_value); 745 746 /* phy_adr_tsel_enable_ 1bit DENALI_PHY_518/646/774 offset_8 */ 747 reg_value = tsel_wr_en << 8; 748 clrsetbits_le32(&denali_phy[518], 0x1 << 8, reg_value); 749 clrsetbits_le32(&denali_phy[646], 0x1 << 8, reg_value); 750 clrsetbits_le32(&denali_phy[774], 0x1 << 8, reg_value); 751 752 /* phy_pad_addr_term tsel 1bit DENALI_PHY_933 offset_17 */ 753 reg_value = tsel_wr_en << 17; 754 clrsetbits_le32(&denali_phy[933], 0x1 << 17, reg_value); 755 /* 756 * pad_rst/cke/cs/clk_term tsel 1bits 757 * DENALI_PHY_938/936/940/934 offset_17 758 */ 759 clrsetbits_le32(&denali_phy[938], 0x1 << 17, reg_value); 760 clrsetbits_le32(&denali_phy[936], 0x1 << 17, reg_value); 761 clrsetbits_le32(&denali_phy[940], 0x1 << 17, reg_value); 762 clrsetbits_le32(&denali_phy[934], 0x1 << 17, reg_value); 763 764 /* phy_pad_fdbk_term 1bit DENALI_PHY_930 offset_17 */ 765 clrsetbits_le32(&denali_phy[930], 0x1 << 17, reg_value); 766 767 if (sdram_params->base.dramtype == LPDDR4) 768 phy_io_config(chan, sdram_params, io->rd_vref, b_reg, channel); 769 else 770 phy_io_config(chan, sdram_params, 0, b_reg, channel); 771 } 772 773 static void pctl_start(struct dram_info *dram, 774 struct rk3399_sdram_params *sdram_params, 775 u32 channel_mask) 776 { 777 const struct chan_info *chan_0 = &dram->chan[0]; 778 const struct chan_info *chan_1 = &dram->chan[1]; 779 780 u32 *denali_ctl_0 = chan_0->pctl->denali_ctl; 781 u32 *denali_phy_0 = chan_0->publ->denali_phy; 782 u32 *denali_ctl_1 = chan_1->pctl->denali_ctl; 783 u32 *denali_phy_1 = chan_1->publ->denali_phy; 784 u32 count, byte, tmp; 785 786 /* PHY_DLL_RST_EN */ 787 if (channel_mask & 1) { 788 writel(0x01000000, &dram->grf->ddrc0_con0); 789 clrsetbits_le32(&denali_phy_0[957], 0x3 << 24, 0x2 << 24); 790 } 791 792 if (channel_mask & 1) { 793 count = 0; 794 while (!(readl(&denali_ctl_0[203]) & (1 << 3))) { 795 if (count > 1000) { 796 printf("channel 0 init err!\n"); 797 while (1) 798 ; 799 } 800 udelay(1); 801 count++; 802 } 803 804 writel(0x01000100, &dram->grf->ddrc0_con0); 805 for (byte = 0; byte < 4; byte++) { 806 tmp = 0x820; 807 writel((tmp << 16) | tmp, 808 &denali_phy_0[53 + (128 * byte)]); 809 writel((tmp << 16) | tmp, 810 &denali_phy_0[54 + (128 * byte)]); 811 writel((tmp << 16) | tmp, 812 &denali_phy_0[55 + (128 * byte)]); 813 writel((tmp << 16) | tmp, 814 &denali_phy_0[56 + (128 * byte)]); 815 writel((tmp << 16) | tmp, 816 &denali_phy_0[57 + (128 * byte)]); 817 clrsetbits_le32(&denali_phy_0[58 + (128 * byte)], 818 0xffff, tmp); 819 } 820 clrsetbits_le32(&denali_ctl_0[68], PWRUP_SREFRESH_EXIT, 821 g_pwrup_srefresh_exit[0]); 822 } 823 824 if (channel_mask & 2) { 825 writel(0x01000000, &dram->grf->ddrc1_con0); 826 clrsetbits_le32(&denali_phy_1[957], 0x3 << 24, 0x2 << 24); 827 } 828 if (channel_mask & 2) { 829 count = 0; 830 while (!(readl(&denali_ctl_1[203]) & (1 << 3))) { 831 if (count > 1000) { 832 printf("channel 1 init err!\n"); 833 while (1) 834 ; 835 } 836 udelay(1); 837 count++; 838 } 839 840 writel(0x01000100, &dram->grf->ddrc1_con0); 841 for (byte = 0; byte < 4; byte++) { 842 tmp = 0x820; 843 writel((tmp << 16) | tmp, 844 &denali_phy_1[53 + (128 * byte)]); 845 writel((tmp << 16) | tmp, 846 &denali_phy_1[54 + (128 * byte)]); 847 writel((tmp << 16) | tmp, 848 &denali_phy_1[55 + (128 * byte)]); 849 writel((tmp << 16) | tmp, 850 &denali_phy_1[56 + (128 * byte)]); 851 writel((tmp << 16) | tmp, 852 &denali_phy_1[57 + (128 * byte)]); 853 clrsetbits_le32(&denali_phy_1[58 + (128 * byte)], 854 0xffff, tmp); 855 } 856 857 clrsetbits_le32(&denali_ctl_1[68], PWRUP_SREFRESH_EXIT, 858 g_pwrup_srefresh_exit[1]); 859 860 /* 861 * restore channel 1 RESET original setting 862 * to avoid 240ohm too weak to prevent ESD test 863 */ 864 if (sdram_params->base.dramtype == LPDDR4) 865 clrsetbits_le32(&denali_phy_1[937], 0xff, 866 sdram_params->phy_regs.denali_phy[937] & 867 0xFF); 868 } 869 } 870 871 /* phy_fn = 0, PHY boot freq 872 * phy_fn = 1, PHY index 0 873 * phy_fn = 2, PHY index 1 874 */ 875 static struct rk3399_sdram_params 876 *get_phy_index_params(u32 phy_fn, 877 struct rk3399_sdram_params *sdram_params) 878 { 879 if (phy_fn == 0) 880 return sdram_params; 881 else if (phy_fn == 1) 882 return &dfs_configs[1]; 883 else if (phy_fn == 2) 884 return &dfs_configs[0]; 885 else 886 return NULL; 887 } 888 889 /* 890 * b_reg: indicate whether set phy register 891 * or just set sdram_params. 892 * if b_reg = 0, channel, mr5 are not care 893 */ 894 static void set_lp4_dq_odt(const struct chan_info *chan, 895 struct rk3399_sdram_params *sdram_params, u32 ctl_fn, 896 u32 en, u32 b_reg, u32 channel, u32 mr5) 897 { 898 u32 *denali_ctl; 899 u32 *denali_pi; 900 u32 reg_value; 901 struct io_setting *io; 902 903 if (b_reg) { 904 denali_pi = chan->pi->denali_pi; 905 denali_ctl = chan->pctl->denali_ctl; 906 } else { 907 denali_pi = sdram_params->pi_regs.denali_pi; 908 denali_ctl = sdram_params->pctl_regs.denali_ctl; 909 } 910 io = get_io_set(sdram_params, mr5); 911 if (en) 912 reg_value = io->dq_odt; 913 else 914 reg_value = 0; 915 916 switch (ctl_fn) { 917 case 0: 918 clrsetbits_le32(&denali_ctl[139], 0x7 << 24, reg_value << 24); 919 clrsetbits_le32(&denali_ctl[153], 0x7 << 24, reg_value << 24); 920 921 clrsetbits_le32(&denali_pi[132], 0x7 << 0, (reg_value << 0)); 922 clrsetbits_le32(&denali_pi[139], 0x7 << 16, (reg_value << 16)); 923 clrsetbits_le32(&denali_pi[147], 0x7 << 0, (reg_value << 0)); 924 clrsetbits_le32(&denali_pi[154], 0x7 << 16, (reg_value << 16)); 925 break; 926 case 1: 927 clrsetbits_le32(&denali_ctl[140], 0x7 << 0, reg_value << 0); 928 clrsetbits_le32(&denali_ctl[154], 0x7 << 0, reg_value << 0); 929 930 clrsetbits_le32(&denali_pi[129], 0x7 << 16, (reg_value << 16)); 931 clrsetbits_le32(&denali_pi[137], 0x7 << 0, (reg_value << 0)); 932 clrsetbits_le32(&denali_pi[144], 0x7 << 16, (reg_value << 16)); 933 clrsetbits_le32(&denali_pi[152], 0x7 << 0, (reg_value << 0)); 934 break; 935 case 2: 936 default: 937 clrsetbits_le32(&denali_ctl[140], 0x7 << 8, (reg_value << 8)); 938 clrsetbits_le32(&denali_ctl[154], 0x7 << 8, (reg_value << 8)); 939 940 clrsetbits_le32(&denali_pi[127], 0x7 << 0, (reg_value << 0)); 941 clrsetbits_le32(&denali_pi[134], 0x7 << 16, (reg_value << 16)); 942 clrsetbits_le32(&denali_pi[142], 0x7 << 0, (reg_value << 0)); 943 clrsetbits_le32(&denali_pi[149], 0x7 << 16, (reg_value << 16)); 944 break; 945 } 946 } 947 948 /* 949 * b_reg: indicate whether set phy register 950 * or just set sdram_params. 951 * if b_reg = 0, channel, mr5 are not care 952 */ 953 static void set_lp4_ca_odt(const struct chan_info *chan, 954 struct rk3399_sdram_params *sdram_params, u32 ctl_fn, 955 u32 en, u32 b_reg, u32 channel, u32 mr5) 956 { 957 u32 *denali_ctl; 958 u32 *denali_pi; 959 u32 reg_value; 960 struct io_setting *io; 961 962 if (b_reg) { 963 denali_pi = chan->pi->denali_pi; 964 denali_ctl = chan->pctl->denali_ctl; 965 } else { 966 denali_pi = sdram_params->pi_regs.denali_pi; 967 denali_ctl = sdram_params->pctl_regs.denali_ctl; 968 } 969 io = get_io_set(sdram_params, mr5); 970 if (en) 971 reg_value = io->ca_odt; 972 else 973 reg_value = 0; 974 975 switch (ctl_fn) { 976 case 0: 977 clrsetbits_le32(&denali_ctl[139], 0x7 << 28, reg_value << 28); 978 clrsetbits_le32(&denali_ctl[153], 0x7 << 28, reg_value << 28); 979 980 clrsetbits_le32(&denali_pi[132], 0x7 << 4, reg_value << 4); 981 clrsetbits_le32(&denali_pi[139], 0x7 << 20, reg_value << 20); 982 clrsetbits_le32(&denali_pi[147], 0x7 << 4, reg_value << 4); 983 clrsetbits_le32(&denali_pi[154], 0x7 << 20, reg_value << 20); 984 break; 985 case 1: 986 clrsetbits_le32(&denali_ctl[140], 0x7 << 4, reg_value << 4); 987 clrsetbits_le32(&denali_ctl[154], 0x7 << 4, reg_value << 4); 988 989 clrsetbits_le32(&denali_pi[129], 0x7 << 20, reg_value << 20); 990 clrsetbits_le32(&denali_pi[137], 0x7 << 4, reg_value << 4); 991 clrsetbits_le32(&denali_pi[144], 0x7 << 20, reg_value << 20); 992 clrsetbits_le32(&denali_pi[152], 0x7 << 4, reg_value << 4); 993 break; 994 case 2: 995 default: 996 clrsetbits_le32(&denali_ctl[140], 0x7 << 12, (reg_value << 12)); 997 clrsetbits_le32(&denali_ctl[154], 0x7 << 12, (reg_value << 12)); 998 999 clrsetbits_le32(&denali_pi[127], 0x7 << 4, reg_value << 4); 1000 clrsetbits_le32(&denali_pi[134], 0x7 << 20, reg_value << 20); 1001 clrsetbits_le32(&denali_pi[142], 0x7 << 4, reg_value << 4); 1002 clrsetbits_le32(&denali_pi[149], 0x7 << 20, reg_value << 20); 1003 break; 1004 } 1005 } 1006 1007 /* 1008 * b_reg: indicate whether set phy register 1009 * or just set sdram_params. 1010 * if b_reg = 0, channel, mr5 are not care 1011 */ 1012 static void set_lp4_MR3(const struct chan_info *chan, 1013 struct rk3399_sdram_params *sdram_params, u32 ctl_fn, 1014 u32 b_reg, u32 channel, u32 mr5) 1015 { 1016 u32 *denali_ctl; 1017 u32 *denali_pi; 1018 u32 reg_value; 1019 struct io_setting *io; 1020 1021 if (b_reg) { 1022 denali_pi = chan->pi->denali_pi; 1023 denali_ctl = chan->pctl->denali_ctl; 1024 } else { 1025 denali_pi = sdram_params->pi_regs.denali_pi; 1026 denali_ctl = sdram_params->pctl_regs.denali_ctl; 1027 } 1028 io = get_io_set(sdram_params, mr5); 1029 1030 reg_value = ((io->pdds << 3) | 1); 1031 switch (ctl_fn) { 1032 case 0: 1033 clrsetbits_le32(&denali_ctl[138], 0xFFFF, reg_value); 1034 clrsetbits_le32(&denali_ctl[152], 0xFFFF, reg_value); 1035 1036 clrsetbits_le32(&denali_pi[131], 0xFFFF << 16, reg_value << 16); 1037 clrsetbits_le32(&denali_pi[139], 0xFFFF, reg_value); 1038 clrsetbits_le32(&denali_pi[146], 0xFFFF << 16, reg_value << 16); 1039 clrsetbits_le32(&denali_pi[154], 0xFFFF, reg_value); 1040 break; 1041 case 1: 1042 clrsetbits_le32(&denali_ctl[138], 0xFFFF << 16, 1043 reg_value << 16); 1044 clrsetbits_le32(&denali_ctl[152], 0xFFFF << 16, 1045 reg_value << 16); 1046 1047 clrsetbits_le32(&denali_pi[129], 0xFFFF, reg_value); 1048 clrsetbits_le32(&denali_pi[136], 0xFFFF << 16, reg_value << 16); 1049 clrsetbits_le32(&denali_pi[144], 0xFFFF, reg_value); 1050 clrsetbits_le32(&denali_pi[151], 0xFFFF << 16, reg_value << 16); 1051 break; 1052 case 2: 1053 default: 1054 clrsetbits_le32(&denali_ctl[139], 0xFFFF, reg_value); 1055 clrsetbits_le32(&denali_ctl[153], 0xFFFF, reg_value); 1056 1057 clrsetbits_le32(&denali_pi[126], 0xFFFF << 16, reg_value << 16); 1058 clrsetbits_le32(&denali_pi[134], 0xFFFF, reg_value); 1059 clrsetbits_le32(&denali_pi[141], 0xFFFF << 16, reg_value << 16); 1060 clrsetbits_le32(&denali_pi[149], 0xFFFF, reg_value); 1061 break; 1062 } 1063 } 1064 1065 /* 1066 * b_reg: indicate whether set phy register 1067 * or just set sdram_params. 1068 * if b_reg = 0, channel, mr5 are not care 1069 */ 1070 static void set_lp4_MR12(const struct chan_info *chan, 1071 struct rk3399_sdram_params *sdram_params, u32 ctl_fn, 1072 u32 b_reg, u32 channel, u32 mr5) 1073 { 1074 u32 *denali_ctl; 1075 u32 *denali_pi; 1076 u32 reg_value; 1077 struct io_setting *io; 1078 1079 if (b_reg) { 1080 denali_pi = chan->pi->denali_pi; 1081 denali_ctl = chan->pctl->denali_ctl; 1082 } else { 1083 denali_pi = sdram_params->pi_regs.denali_pi; 1084 denali_ctl = sdram_params->pctl_regs.denali_ctl; 1085 } 1086 io = get_io_set(sdram_params, mr5); 1087 1088 reg_value = io->ca_vref; 1089 switch (ctl_fn) { 1090 case 0: 1091 clrsetbits_le32(&denali_ctl[140], 0xFFFF << 16, 1092 reg_value << 16); 1093 clrsetbits_le32(&denali_ctl[154], 0xFFFF << 16, 1094 reg_value << 16); 1095 1096 clrsetbits_le32(&denali_pi[132], 0xFF << 8, reg_value << 8); 1097 clrsetbits_le32(&denali_pi[139], 0xFF << 24, reg_value << 24); 1098 clrsetbits_le32(&denali_pi[147], 0xFF << 8, reg_value << 8); 1099 clrsetbits_le32(&denali_pi[154], 0xFF << 24, reg_value << 24); 1100 break; 1101 case 1: 1102 clrsetbits_le32(&denali_ctl[141], 0xFFFF, reg_value); 1103 clrsetbits_le32(&denali_ctl[155], 0xFFFF, reg_value); 1104 1105 clrsetbits_le32(&denali_pi[129], 0xFF << 24, reg_value << 24); 1106 clrsetbits_le32(&denali_pi[137], 0xFF << 8, reg_value << 8); 1107 clrsetbits_le32(&denali_pi[144], 0xFF << 24, reg_value << 24); 1108 clrsetbits_le32(&denali_pi[152], 0xFF << 8, reg_value << 8); 1109 break; 1110 case 2: 1111 default: 1112 clrsetbits_le32(&denali_ctl[141], 0xFFFF << 16, 1113 reg_value << 16); 1114 clrsetbits_le32(&denali_ctl[155], 0xFFFF << 16, 1115 reg_value << 16); 1116 1117 clrsetbits_le32(&denali_pi[127], 0xFF << 8, reg_value << 8); 1118 clrsetbits_le32(&denali_pi[134], 0xFF << 24, reg_value << 24); 1119 clrsetbits_le32(&denali_pi[142], 0xFF << 8, reg_value << 8); 1120 clrsetbits_le32(&denali_pi[149], 0xFF << 24, reg_value << 24); 1121 break; 1122 } 1123 } 1124 1125 /* 1126 * b_reg: indicate whether set phy register 1127 * or just set sdram_params. 1128 * if b_reg = 0, channel, mr5 are not care 1129 */ 1130 static void set_lp4_MR14(const struct chan_info *chan, 1131 struct rk3399_sdram_params *sdram_params, u32 ctl_fn, 1132 u32 b_reg, u32 channel, u32 mr5) 1133 { 1134 u32 *denali_ctl; 1135 u32 *denali_pi; 1136 u32 reg_value; 1137 struct io_setting *io; 1138 1139 if (b_reg) { 1140 denali_pi = chan->pi->denali_pi; 1141 denali_ctl = chan->pctl->denali_ctl; 1142 } else { 1143 denali_pi = sdram_params->pi_regs.denali_pi; 1144 denali_ctl = sdram_params->pctl_regs.denali_ctl; 1145 } 1146 io = get_io_set(sdram_params, mr5); 1147 1148 reg_value = io->dq_vref; 1149 switch (ctl_fn) { 1150 case 0: 1151 clrsetbits_le32(&denali_ctl[142], 0xFFFF << 16, 1152 reg_value << 16); 1153 clrsetbits_le32(&denali_ctl[156], 0xFFFF << 16, 1154 reg_value << 16); 1155 1156 clrsetbits_le32(&denali_pi[132], 0xFF << 16, reg_value << 16); 1157 clrsetbits_le32(&denali_pi[140], 0xFF << 0, reg_value << 0); 1158 clrsetbits_le32(&denali_pi[147], 0xFF << 16, reg_value << 16); 1159 clrsetbits_le32(&denali_pi[155], 0xFF << 0, reg_value << 0); 1160 break; 1161 case 1: 1162 clrsetbits_le32(&denali_ctl[143], 0xFFFF, reg_value); 1163 clrsetbits_le32(&denali_ctl[157], 0xFFFF, reg_value); 1164 1165 clrsetbits_le32(&denali_pi[130], 0xFF << 0, reg_value << 0); 1166 clrsetbits_le32(&denali_pi[137], 0xFF << 16, reg_value << 16); 1167 clrsetbits_le32(&denali_pi[145], 0xFF << 0, reg_value << 0); 1168 clrsetbits_le32(&denali_pi[152], 0xFF << 16, reg_value << 16); 1169 break; 1170 case 2: 1171 default: 1172 clrsetbits_le32(&denali_ctl[143], 0xFFFF << 16, 1173 reg_value << 16); 1174 clrsetbits_le32(&denali_ctl[157], 0xFFFF << 16, 1175 reg_value << 16); 1176 1177 clrsetbits_le32(&denali_pi[127], 0xFF << 16, reg_value << 16); 1178 clrsetbits_le32(&denali_pi[135], 0xFF << 0, reg_value << 0); 1179 clrsetbits_le32(&denali_pi[142], 0xFF << 16, reg_value << 16); 1180 clrsetbits_le32(&denali_pi[150], 0xFF << 0, reg_value << 0); 1181 break; 1182 } 1183 } 1184 1185 static void modify_param(const struct chan_info *chan, 1186 struct rk3399_sdram_params *sdram_params) 1187 { 1188 struct rk3399_sdram_params *params; 1189 u32 *denali_ctl_params; 1190 u32 *denali_pi_params; 1191 u32 *denali_phy_params; 1192 1193 denali_ctl_params = sdram_params->pctl_regs.denali_ctl; 1194 denali_pi_params = sdram_params->pi_regs.denali_pi; 1195 denali_phy_params = sdram_params->phy_regs.denali_phy; 1196 1197 if (sdram_params->base.dramtype == LPDDR4) { 1198 set_lp4_dq_odt(chan, sdram_params, 2, 1, 0, 0, 0); 1199 set_lp4_ca_odt(chan, sdram_params, 2, 1, 0, 0, 0); 1200 set_lp4_MR3(chan, sdram_params, 2, 0, 0, 0); 1201 set_lp4_MR12(chan, sdram_params, 2, 0, 0, 0); 1202 set_lp4_MR14(chan, sdram_params, 2, 0, 0, 0); 1203 params = get_phy_index_params(0, sdram_params); 1204 set_ds_odt(chan, params, 0, 0, 0); 1205 /* read two cycle preamble */ 1206 clrsetbits_le32(&denali_ctl_params[200], 0x3 << 24, 0x3 << 24); 1207 clrsetbits_le32(&denali_phy_params[7], 0x3 << 24, 0x3 << 24); 1208 clrsetbits_le32(&denali_phy_params[135], 0x3 << 24, 0x3 << 24); 1209 clrsetbits_le32(&denali_phy_params[263], 0x3 << 24, 0x3 << 24); 1210 clrsetbits_le32(&denali_phy_params[391], 0x3 << 24, 0x3 << 24); 1211 1212 /* boot frequency two cycle preamble */ 1213 clrsetbits_le32(&denali_phy_params[2], 0x3 << 16, 0x3 << 16); 1214 clrsetbits_le32(&denali_phy_params[130], 0x3 << 16, 0x3 << 16); 1215 clrsetbits_le32(&denali_phy_params[258], 0x3 << 16, 0x3 << 16); 1216 clrsetbits_le32(&denali_phy_params[386], 0x3 << 16, 0x3 << 16); 1217 1218 clrsetbits_le32(&denali_pi_params[45], 0x3 << 8, 0x3 << 8); 1219 clrsetbits_le32(&denali_pi_params[58], 0x1, 0x1); 1220 1221 /* 1222 * bypass mode need PHY_SLICE_PWR_RDC_DISABLE_x = 1, 1223 * boot frequency mode use bypass mode 1224 */ 1225 setbits_le32(&denali_phy_params[10], 1 << 16); 1226 setbits_le32(&denali_phy_params[138], 1 << 16); 1227 setbits_le32(&denali_phy_params[266], 1 << 16); 1228 setbits_le32(&denali_phy_params[394], 1 << 16); 1229 } else { 1230 /* modify PHY F0/F1/F2 params */ 1231 params = get_phy_index_params(0, sdram_params); 1232 set_ds_odt(chan, params, 0, 0, 0); 1233 } 1234 1235 clrsetbits_le32(&denali_pi_params[45], 0x1 << 24, 0x1 << 24); 1236 clrsetbits_le32(&denali_pi_params[61], 0x1 << 24, 0x1 << 24); 1237 clrsetbits_le32(&denali_pi_params[76], 0x1 << 24, 0x1 << 24); 1238 clrsetbits_le32(&denali_pi_params[77], 0x1, 0x1); 1239 } 1240 1241 static int pctl_cfg(const struct chan_info *chan, u32 channel, 1242 struct rk3399_sdram_params *sdram_params) 1243 { 1244 u32 *denali_ctl = chan->pctl->denali_ctl; 1245 u32 *denali_pi = chan->pi->denali_pi; 1246 u32 *denali_phy = chan->publ->denali_phy; 1247 const u32 *params_ctl = sdram_params->pctl_regs.denali_ctl; 1248 const u32 *params_phy = sdram_params->phy_regs.denali_phy; 1249 u32 tmp, tmp1, tmp2; 1250 struct rk3399_sdram_params *params; 1251 u32 byte; 1252 1253 modify_param(chan, sdram_params); 1254 /* 1255 * work around controller bug: 1256 * Do not program DRAM_CLASS until NO_PHY_IND_TRAIN_INT is programmed 1257 */ 1258 sdram_copy_to_reg(&denali_ctl[1], ¶ms_ctl[1], 1259 sizeof(struct rk3399_ddr_pctl_regs) - 4); 1260 writel(params_ctl[0], &denali_ctl[0]); 1261 1262 /* 1263 * two channel init at the same time, then ZQ Cal Start 1264 * at the same time, it will use the same RZQ. 1265 * to fix it: increase tINIT3 for channel 1, will avoid two 1266 * channel ZQ Cal Start at the same time 1267 */ 1268 if (sdram_params->base.dramtype == LPDDR4 && channel == 1) { 1269 tmp = ((1000000 * (sdram_params->base.ddr_freq / MHz) + 999) / 1270 1000); 1271 tmp1 = readl(&denali_ctl[14]); 1272 writel(tmp + tmp1, &denali_ctl[14]); 1273 } 1274 1275 sdram_copy_to_reg(denali_pi, &sdram_params->pi_regs.denali_pi[0], 1276 sizeof(struct rk3399_ddr_pi_regs)); 1277 /* rank count need to set for init */ 1278 set_memory_map(chan, channel, sdram_params); 1279 1280 writel(sdram_params->phy_regs.denali_phy[910], &denali_phy[910]); 1281 writel(sdram_params->phy_regs.denali_phy[911], &denali_phy[911]); 1282 writel(sdram_params->phy_regs.denali_phy[912], &denali_phy[912]); 1283 1284 if (sdram_params->base.dramtype == LPDDR4) { 1285 writel(sdram_params->phy_regs.denali_phy[898], 1286 &denali_phy[898]); 1287 writel(sdram_params->phy_regs.denali_phy[919], 1288 &denali_phy[919]); 1289 } 1290 1291 g_pwrup_srefresh_exit[channel] = readl(&denali_ctl[68]) & 1292 PWRUP_SREFRESH_EXIT; 1293 clrbits_le32(&denali_ctl[68], PWRUP_SREFRESH_EXIT); 1294 1295 /* PHY_DLL_RST_EN */ 1296 clrsetbits_le32(&denali_phy[957], 0x3 << 24, 1 << 24); 1297 1298 setbits_le32(&denali_pi[0], START); 1299 setbits_le32(&denali_ctl[0], START); 1300 1301 /* because LPDDR4 use PLL bypass mode for init 1302 * not need to wait for the PLL to lock 1303 */ 1304 if (sdram_params->base.dramtype != LPDDR4) { 1305 /* wait lock */ 1306 while (1) { 1307 tmp = readl(&denali_phy[920]); 1308 tmp1 = readl(&denali_phy[921]); 1309 tmp2 = readl(&denali_phy[922]); 1310 if ((((tmp >> 16) & 0x1) == 0x1) && 1311 (((tmp1 >> 16) & 0x1) == 0x1) && 1312 (((tmp1 >> 0) & 0x1) == 0x1) && 1313 (((tmp2 >> 0) & 0x1) == 0x1)) 1314 break; 1315 } 1316 } 1317 1318 sdram_copy_to_reg(&denali_phy[896], ¶ms_phy[896], (958 - 895) * 4); 1319 sdram_copy_to_reg(&denali_phy[0], ¶ms_phy[0], (90 - 0 + 1) * 4); 1320 sdram_copy_to_reg(&denali_phy[128], 1321 ¶ms_phy[128], (218 - 128 + 1) * 4); 1322 sdram_copy_to_reg(&denali_phy[256], 1323 ¶ms_phy[256], (346 - 256 + 1) * 4); 1324 sdram_copy_to_reg(&denali_phy[384], 1325 ¶ms_phy[384], (474 - 384 + 1) * 4); 1326 sdram_copy_to_reg(&denali_phy[512], 1327 ¶ms_phy[512], (549 - 512 + 1) * 4); 1328 sdram_copy_to_reg(&denali_phy[640], 1329 ¶ms_phy[640], (677 - 640 + 1) * 4); 1330 sdram_copy_to_reg(&denali_phy[768], 1331 ¶ms_phy[768], (805 - 768 + 1) * 4); 1332 1333 if (sdram_params->base.dramtype == LPDDR4) 1334 params = get_phy_index_params(1, sdram_params); 1335 else 1336 params = get_phy_index_params(0, sdram_params); 1337 1338 clrsetbits_le32(¶ms->phy_regs.denali_phy[896], 0x3 << 8, 1339 0 << 8); 1340 writel(params->phy_regs.denali_phy[896], &denali_phy[896]); 1341 1342 writel(sdram_params->phy_regs.denali_phy[83] + (0x10 << 16), 1343 &denali_phy[83]); 1344 writel(sdram_params->phy_regs.denali_phy[84] + (0x10 << 8), 1345 &denali_phy[84]); 1346 writel(sdram_params->phy_regs.denali_phy[211] + (0x10 << 16), 1347 &denali_phy[211]); 1348 writel(sdram_params->phy_regs.denali_phy[212] + (0x10 << 8), 1349 &denali_phy[212]); 1350 writel(sdram_params->phy_regs.denali_phy[339] + (0x10 << 16), 1351 &denali_phy[339]); 1352 writel(sdram_params->phy_regs.denali_phy[340] + (0x10 << 8), 1353 &denali_phy[340]); 1354 writel(sdram_params->phy_regs.denali_phy[467] + (0x10 << 16), 1355 &denali_phy[467]); 1356 writel(sdram_params->phy_regs.denali_phy[468] + (0x10 << 8), 1357 &denali_phy[468]); 1358 1359 if (sdram_params->base.dramtype == LPDDR4) { 1360 /* 1361 * to improve write dqs and dq phase from 1.5ns to 3.5ns 1362 * at 50MHz. 1363 */ 1364 for (byte = 0; byte < 4; byte++) { 1365 tmp = 0x680; 1366 clrsetbits_le32(&denali_phy[1 + (128 * byte)], 1367 0xfff << 8, tmp << 8); 1368 } 1369 /* 1370 * to workaround 366ball two channel's RESET connect to 1371 * one RESET signal of die 1372 */ 1373 if (channel == 1) 1374 clrsetbits_le32(&denali_phy[937], 0xff, 1375 PHY_DRV_ODT_240 | 1376 (PHY_DRV_ODT_240 << 0x4)); 1377 } 1378 1379 return 0; 1380 } 1381 1382 static void select_per_cs_training_index(const struct chan_info *chan, 1383 u32 rank) 1384 { 1385 u32 *denali_phy = chan->publ->denali_phy; 1386 1387 /* PHY_84 PHY_PER_CS_TRAINING_EN_0 1bit offset_16 */ 1388 if ((readl(&denali_phy[84]) >> 16) & 1) { 1389 /* 1390 * PHY_8/136/264/392 1391 * phy_per_cs_training_index_X 1bit offset_24 1392 */ 1393 clrsetbits_le32(&denali_phy[8], 0x1 << 24, rank << 24); 1394 clrsetbits_le32(&denali_phy[136], 0x1 << 24, rank << 24); 1395 clrsetbits_le32(&denali_phy[264], 0x1 << 24, rank << 24); 1396 clrsetbits_le32(&denali_phy[392], 0x1 << 24, rank << 24); 1397 } 1398 } 1399 1400 static void override_write_leveling_value(const struct chan_info *chan) 1401 { 1402 u32 *denali_ctl = chan->pctl->denali_ctl; 1403 u32 *denali_phy = chan->publ->denali_phy; 1404 u32 byte; 1405 1406 /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */ 1407 setbits_le32(&denali_phy[896], 1); 1408 1409 /* 1410 * PHY_8/136/264/392 1411 * phy_per_cs_training_multicast_en_X 1bit offset_16 1412 */ 1413 clrsetbits_le32(&denali_phy[8], 0x1 << 16, 1 << 16); 1414 clrsetbits_le32(&denali_phy[136], 0x1 << 16, 1 << 16); 1415 clrsetbits_le32(&denali_phy[264], 0x1 << 16, 1 << 16); 1416 clrsetbits_le32(&denali_phy[392], 0x1 << 16, 1 << 16); 1417 1418 for (byte = 0; byte < 4; byte++) 1419 clrsetbits_le32(&denali_phy[63 + (128 * byte)], 0xffff << 16, 1420 0x200 << 16); 1421 1422 /* PHY_896 PHY_FREQ_SEL_MULTICAST_EN 1bit offset_0 */ 1423 clrbits_le32(&denali_phy[896], 1); 1424 1425 /* CTL_200 ctrlupd_req 1bit offset_8 */ 1426 clrsetbits_le32(&denali_ctl[200], 0x1 << 8, 0x1 << 8); 1427 } 1428 1429 static int data_training_ca(const struct chan_info *chan, u32 channel, 1430 const struct rk3399_sdram_params *sdram_params) 1431 { 1432 u32 *denali_pi = chan->pi->denali_pi; 1433 u32 *denali_phy = chan->publ->denali_phy; 1434 u32 i, tmp; 1435 u32 obs_0, obs_1, obs_2, obs_err = 0; 1436 u32 rank = sdram_params->ch[channel].cap_info.rank; 1437 u32 rank_mask; 1438 1439 if (sdram_params->base.dramtype == LPDDR4) 1440 rank_mask = (rank == 1) ? 0x5 : 0xf; 1441 else 1442 rank_mask = (rank == 1) ? 0x1 : 0x3; 1443 1444 for (i = 0; i < 4; i++) { 1445 if (!(rank_mask & (1 << i))) 1446 continue; 1447 select_per_cs_training_index(chan, i); 1448 /* PI_100 PI_CALVL_EN:RW:8:2 */ 1449 clrsetbits_le32(&denali_pi[100], 0x3 << 8, 0x2 << 8); 1450 /* PI_92 PI_CALVL_REQ:WR:16:1,PI_CALVL_CS:RW:24:2 */ 1451 clrsetbits_le32(&denali_pi[92], 1452 (0x1 << 16) | (0x3 << 24), 1453 (0x1 << 16) | (i << 24)); 1454 1455 /* Waiting for training complete */ 1456 while (1) { 1457 /* PI_174 PI_INT_STATUS:RD:8:18 */ 1458 tmp = readl(&denali_pi[174]) >> 8; 1459 /* 1460 * check status obs 1461 * PHY_532/660/789 phy_adr_calvl_obs1_:0:32 1462 */ 1463 obs_0 = readl(&denali_phy[532]); 1464 obs_1 = readl(&denali_phy[660]); 1465 obs_2 = readl(&denali_phy[788]); 1466 if (((obs_0 >> 30) & 0x3) || 1467 ((obs_1 >> 30) & 0x3) || 1468 ((obs_2 >> 30) & 0x3)) 1469 obs_err = 1; 1470 if ((((tmp >> 11) & 0x1) == 0x1) && 1471 (((tmp >> 13) & 0x1) == 0x1) && 1472 (((tmp >> 5) & 0x1) == 0x0) && 1473 obs_err == 0) 1474 break; 1475 else if ((((tmp >> 5) & 0x1) == 0x1) || 1476 (obs_err == 1)) 1477 return -EIO; 1478 } 1479 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 1480 writel(0x00003f7c, (&denali_pi[175])); 1481 } 1482 clrbits_le32(&denali_pi[100], 0x3 << 8); 1483 1484 return 0; 1485 } 1486 1487 static int data_training_wl(const struct chan_info *chan, u32 channel, 1488 const struct rk3399_sdram_params *sdram_params) 1489 { 1490 u32 *denali_pi = chan->pi->denali_pi; 1491 u32 *denali_phy = chan->publ->denali_phy; 1492 u32 i, tmp; 1493 u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0; 1494 u32 rank = sdram_params->ch[channel].cap_info.rank; 1495 1496 for (i = 0; i < rank; i++) { 1497 select_per_cs_training_index(chan, i); 1498 /* PI_60 PI_WRLVL_EN:RW:8:2 */ 1499 clrsetbits_le32(&denali_pi[60], 0x3 << 8, 0x2 << 8); 1500 /* PI_59 PI_WRLVL_REQ:WR:8:1,PI_WRLVL_CS:RW:16:2 */ 1501 clrsetbits_le32(&denali_pi[59], 1502 (0x1 << 8) | (0x3 << 16), 1503 (0x1 << 8) | (i << 16)); 1504 1505 /* Waiting for training complete */ 1506 while (1) { 1507 /* PI_174 PI_INT_STATUS:RD:8:18 */ 1508 tmp = readl(&denali_pi[174]) >> 8; 1509 1510 /* 1511 * check status obs, if error maybe can not 1512 * get leveling done PHY_40/168/296/424 1513 * phy_wrlvl_status_obs_X:0:13 1514 */ 1515 obs_0 = readl(&denali_phy[40]); 1516 obs_1 = readl(&denali_phy[168]); 1517 obs_2 = readl(&denali_phy[296]); 1518 obs_3 = readl(&denali_phy[424]); 1519 if (((obs_0 >> 12) & 0x1) || 1520 ((obs_1 >> 12) & 0x1) || 1521 ((obs_2 >> 12) & 0x1) || 1522 ((obs_3 >> 12) & 0x1)) 1523 obs_err = 1; 1524 if ((((tmp >> 10) & 0x1) == 0x1) && 1525 (((tmp >> 13) & 0x1) == 0x1) && 1526 (((tmp >> 4) & 0x1) == 0x0) && 1527 obs_err == 0) 1528 break; 1529 else if ((((tmp >> 4) & 0x1) == 0x1) || 1530 (obs_err == 1)) 1531 return -EIO; 1532 } 1533 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 1534 writel(0x00003f7c, (&denali_pi[175])); 1535 } 1536 1537 override_write_leveling_value(chan); 1538 clrbits_le32(&denali_pi[60], 0x3 << 8); 1539 1540 return 0; 1541 } 1542 1543 static int data_training_rg(const struct chan_info *chan, u32 channel, 1544 const struct rk3399_sdram_params *sdram_params) 1545 { 1546 u32 *denali_pi = chan->pi->denali_pi; 1547 u32 *denali_phy = chan->publ->denali_phy; 1548 u32 i, tmp; 1549 u32 obs_0, obs_1, obs_2, obs_3, obs_err = 0; 1550 u32 rank = sdram_params->ch[channel].cap_info.rank; 1551 1552 for (i = 0; i < rank; i++) { 1553 select_per_cs_training_index(chan, i); 1554 /* PI_80 PI_RDLVL_GATE_EN:RW:24:2 */ 1555 clrsetbits_le32(&denali_pi[80], 0x3 << 24, 0x2 << 24); 1556 /* 1557 * PI_74 PI_RDLVL_GATE_REQ:WR:16:1 1558 * PI_RDLVL_CS:RW:24:2 1559 */ 1560 clrsetbits_le32(&denali_pi[74], 1561 (0x1 << 16) | (0x3 << 24), 1562 (0x1 << 16) | (i << 24)); 1563 1564 /* Waiting for training complete */ 1565 while (1) { 1566 /* PI_174 PI_INT_STATUS:RD:8:18 */ 1567 tmp = readl(&denali_pi[174]) >> 8; 1568 1569 /* 1570 * check status obs 1571 * PHY_43/171/299/427 1572 * PHY_GTLVL_STATUS_OBS_x:16:8 1573 */ 1574 obs_0 = readl(&denali_phy[43]); 1575 obs_1 = readl(&denali_phy[171]); 1576 obs_2 = readl(&denali_phy[299]); 1577 obs_3 = readl(&denali_phy[427]); 1578 if (((obs_0 >> (16 + 6)) & 0x3) || 1579 ((obs_1 >> (16 + 6)) & 0x3) || 1580 ((obs_2 >> (16 + 6)) & 0x3) || 1581 ((obs_3 >> (16 + 6)) & 0x3)) 1582 obs_err = 1; 1583 if ((((tmp >> 9) & 0x1) == 0x1) && 1584 (((tmp >> 13) & 0x1) == 0x1) && 1585 (((tmp >> 3) & 0x1) == 0x0) && 1586 obs_err == 0) 1587 break; 1588 else if ((((tmp >> 3) & 0x1) == 0x1) || 1589 (obs_err == 1)) 1590 return -EIO; 1591 } 1592 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 1593 writel(0x00003f7c, (&denali_pi[175])); 1594 } 1595 clrbits_le32(&denali_pi[80], 0x3 << 24); 1596 1597 return 0; 1598 } 1599 1600 static int data_training_rl(const struct chan_info *chan, u32 channel, 1601 const struct rk3399_sdram_params *sdram_params) 1602 { 1603 u32 *denali_pi = chan->pi->denali_pi; 1604 u32 i, tmp; 1605 u32 rank = sdram_params->ch[channel].cap_info.rank; 1606 1607 for (i = 0; i < rank; i++) { 1608 select_per_cs_training_index(chan, i); 1609 /* PI_80 PI_RDLVL_EN:RW:16:2 */ 1610 clrsetbits_le32(&denali_pi[80], 0x3 << 16, 0x2 << 16); 1611 /* PI_74 PI_RDLVL_REQ:WR:8:1,PI_RDLVL_CS:RW:24:2 */ 1612 clrsetbits_le32(&denali_pi[74], 1613 (0x1 << 8) | (0x3 << 24), 1614 (0x1 << 8) | (i << 24)); 1615 1616 /* Waiting for training complete */ 1617 while (1) { 1618 /* PI_174 PI_INT_STATUS:RD:8:18 */ 1619 tmp = readl(&denali_pi[174]) >> 8; 1620 1621 /* 1622 * make sure status obs not report error bit 1623 * PHY_46/174/302/430 1624 * phy_rdlvl_status_obs_X:16:8 1625 */ 1626 if ((((tmp >> 8) & 0x1) == 0x1) && 1627 (((tmp >> 13) & 0x1) == 0x1) && 1628 (((tmp >> 2) & 0x1) == 0x0)) 1629 break; 1630 else if (((tmp >> 2) & 0x1) == 0x1) 1631 return -EIO; 1632 } 1633 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 1634 writel(0x00003f7c, (&denali_pi[175])); 1635 } 1636 clrbits_le32(&denali_pi[80], 0x3 << 16); 1637 1638 return 0; 1639 } 1640 1641 static int data_training_wdql(const struct chan_info *chan, u32 channel, 1642 const struct rk3399_sdram_params *sdram_params) 1643 { 1644 u32 *denali_pi = chan->pi->denali_pi; 1645 u32 i, tmp; 1646 u32 rank = sdram_params->ch[channel].cap_info.rank; 1647 u32 rank_mask; 1648 1649 if (sdram_params->base.dramtype == LPDDR4) 1650 rank_mask = (rank == 1) ? 0x5 : 0xf; 1651 else 1652 rank_mask = (rank == 1) ? 0x1 : 0x3; 1653 1654 for (i = 0; i < 4; i++) { 1655 if (!(rank_mask & (1 << i))) 1656 continue; 1657 1658 select_per_cs_training_index(chan, i); 1659 /* 1660 * disable PI_WDQLVL_VREF_EN before wdq leveling? 1661 * PI_117 PI_WDQLVL_VREF_EN:RW:8:1 1662 */ 1663 clrbits_le32(&denali_pi[117], 0x1 << 8); 1664 /* PI_124 PI_WDQLVL_EN:RW:16:2 */ 1665 clrsetbits_le32(&denali_pi[124], 0x3 << 16, 0x2 << 16); 1666 /* PI_121 PI_WDQLVL_REQ:WR:8:1,PI_WDQLVL_CS:RW:16:2 */ 1667 clrsetbits_le32(&denali_pi[121], 1668 (0x1 << 8) | (0x3 << 16), 1669 (0x1 << 8) | (i << 16)); 1670 1671 /* Waiting for training complete */ 1672 while (1) { 1673 /* PI_174 PI_INT_STATUS:RD:8:18 */ 1674 tmp = readl(&denali_pi[174]) >> 8; 1675 if ((((tmp >> 12) & 0x1) == 0x1) && 1676 (((tmp >> 13) & 0x1) == 0x1) && 1677 (((tmp >> 6) & 0x1) == 0x0)) 1678 break; 1679 else if (((tmp >> 6) & 0x1) == 0x1) 1680 return -EIO; 1681 } 1682 /* clear interrupt,PI_175 PI_INT_ACK:WR:0:17 */ 1683 writel(0x00003f7c, (&denali_pi[175])); 1684 } 1685 clrbits_le32(&denali_pi[124], 0x3 << 16); 1686 1687 return 0; 1688 } 1689 1690 static int data_training(const struct chan_info *chan, u32 channel, 1691 const struct rk3399_sdram_params *sdram_params, 1692 u32 training_flag) 1693 { 1694 u32 *denali_phy = chan->publ->denali_phy; 1695 int ret = 0; 1696 1697 /* PHY_927 PHY_PAD_DQS_DRIVE RPULL offset_22 */ 1698 setbits_le32(&denali_phy[927], (1 << 22)); 1699 1700 if (training_flag == PI_FULL_TRAINING) { 1701 if (sdram_params->base.dramtype == LPDDR4) { 1702 training_flag = PI_WRITE_LEVELING | 1703 PI_READ_GATE_TRAINING | 1704 PI_READ_LEVELING | PI_WDQ_LEVELING; 1705 } else if (sdram_params->base.dramtype == LPDDR3) { 1706 training_flag = PI_CA_TRAINING | PI_WRITE_LEVELING | 1707 PI_READ_GATE_TRAINING; 1708 } else if (sdram_params->base.dramtype == DDR3) { 1709 training_flag = PI_WRITE_LEVELING | 1710 PI_READ_GATE_TRAINING | 1711 PI_READ_LEVELING; 1712 } 1713 } 1714 1715 /* ca training(LPDDR4,LPDDR3 support) */ 1716 if ((training_flag & PI_CA_TRAINING) == PI_CA_TRAINING) { 1717 ret = data_training_ca(chan, channel, sdram_params); 1718 if (ret != 0) 1719 goto out; 1720 } 1721 1722 /* write leveling(LPDDR4,LPDDR3,DDR3 support) */ 1723 if ((training_flag & PI_WRITE_LEVELING) == PI_WRITE_LEVELING) { 1724 ret = data_training_wl(chan, channel, sdram_params); 1725 if (ret != 0) 1726 goto out; 1727 } 1728 1729 /* read gate training(LPDDR4,LPDDR3,DDR3 support) */ 1730 if ((training_flag & PI_READ_GATE_TRAINING) == PI_READ_GATE_TRAINING) { 1731 ret = data_training_rg(chan, channel, sdram_params); 1732 if (ret != 0) 1733 goto out; 1734 } 1735 1736 /* read leveling(LPDDR4,LPDDR3,DDR3 support) */ 1737 if ((training_flag & PI_READ_LEVELING) == PI_READ_LEVELING) { 1738 ret = data_training_rl(chan, channel, sdram_params); 1739 if (ret != 0) 1740 goto out; 1741 } 1742 1743 /* wdq leveling(LPDDR4 support) */ 1744 if ((training_flag & PI_WDQ_LEVELING) == PI_WDQ_LEVELING) { 1745 ret = data_training_wdql(chan, channel, sdram_params); 1746 if (ret != 0) 1747 goto out; 1748 } 1749 1750 /* PHY_927 PHY_PAD_DQS_DRIVE RPULL offset_22 */ 1751 clrbits_le32(&denali_phy[927], (1 << 22)); 1752 1753 out: 1754 return ret; 1755 } 1756 1757 static void set_ddrconfig(const struct chan_info *chan, 1758 const struct rk3399_sdram_params *sdram_params, 1759 unsigned char channel, u32 ddrconfig) 1760 { 1761 /* only need to set ddrconfig */ 1762 struct msch_regs *ddr_msch_regs = chan->msch; 1763 unsigned int cs0_cap = 0; 1764 unsigned int cs1_cap = 0; 1765 1766 cs0_cap = (1 << (sdram_params->ch[channel].cap_info.cs0_row 1767 + sdram_params->ch[channel].cap_info.col 1768 + sdram_params->ch[channel].cap_info.bk 1769 + sdram_params->ch[channel].cap_info.bw - 20)); 1770 if (sdram_params->ch[channel].cap_info.rank > 1) 1771 cs1_cap = cs0_cap >> (sdram_params->ch[channel].cap_info.cs0_row 1772 - sdram_params->ch[channel].cap_info.cs1_row); 1773 if (sdram_params->ch[channel].cap_info.row_3_4) { 1774 cs0_cap = cs0_cap * 3 / 4; 1775 cs1_cap = cs1_cap * 3 / 4; 1776 } 1777 1778 writel(ddrconfig | (ddrconfig << 8), &ddr_msch_regs->ddrconf); 1779 writel(((cs0_cap / 32) & 0xff) | (((cs1_cap / 32) & 0xff) << 8), 1780 &ddr_msch_regs->ddrsize); 1781 } 1782 1783 static void dram_all_config(struct dram_info *dram, 1784 struct rk3399_sdram_params *sdram_params) 1785 { 1786 u32 sys_reg2 = 0; 1787 u32 sys_reg3 = 0; 1788 unsigned int channel, idx; 1789 1790 for (channel = 0, idx = 0; 1791 (idx < sdram_params->base.num_channels) && (channel < 2); 1792 channel++) { 1793 struct msch_regs *ddr_msch_regs; 1794 struct sdram_msch_timings *noc_timing; 1795 1796 if (sdram_params->ch[channel].cap_info.col == 0) 1797 continue; 1798 idx++; 1799 sdram_org_config(&sdram_params->ch[channel].cap_info, 1800 &sdram_params->base, &sys_reg2, 1801 &sys_reg3, channel); 1802 ddr_msch_regs = dram->chan[channel].msch; 1803 noc_timing = &sdram_params->ch[channel].noc_timings; 1804 sdram_msch_config(ddr_msch_regs, noc_timing); 1805 1806 /* rank 1 memory clock disable (dfi_dram_clk_disable = 1) */ 1807 if (sdram_params->ch[channel].cap_info.rank == 1) 1808 setbits_le32(&dram->chan[channel].pctl->denali_ctl[276], 1809 1 << 17); 1810 } 1811 1812 writel(sys_reg2, &dram->pmugrf->os_reg2); 1813 writel(sys_reg3, &dram->pmugrf->os_reg3); 1814 rk_clrsetreg(&dram->pmusgrf->soc_con4, 0x1f << 10, 1815 sdram_params->base.stride << 10); 1816 1817 /* reboot hold register set */ 1818 writel(PRESET_SGRF_HOLD(0) | PRESET_GPIO0_HOLD(1) | 1819 PRESET_GPIO1_HOLD(1), 1820 &dram->pmucru->pmucru_rstnhold_con[1]); 1821 clrsetbits_le32(&dram->cru->glb_rst_con, 0x3, 0x3); 1822 } 1823 1824 static int switch_to_phy_index1(struct dram_info *dram, 1825 const struct rk3399_sdram_params *sdram_params) 1826 { 1827 u32 channel; 1828 u32 *denali_phy; 1829 u32 ch_count = sdram_params->base.num_channels; 1830 int ret; 1831 int i = 0; 1832 1833 writel(RK_CLRSETBITS(0x03 << 4 | 1 << 2 | 1, 1834 1 << 4 | 1 << 2 | 1), 1835 &dram->cic->cic_ctrl0); 1836 while (!(readl(&dram->cic->cic_status0) & (1 << 2))) { 1837 mdelay(10); 1838 i++; 1839 if (i > 10) { 1840 debug("index1 frequency change overtime\n"); 1841 return -ETIME; 1842 } 1843 } 1844 1845 i = 0; 1846 writel(RK_CLRSETBITS(1 << 1, 1 << 1), &dram->cic->cic_ctrl0); 1847 while (!(readl(&dram->cic->cic_status0) & (1 << 0))) { 1848 mdelay(10); 1849 if (i > 10) { 1850 debug("index1 frequency done overtime\n"); 1851 return -ETIME; 1852 } 1853 } 1854 1855 for (channel = 0; channel < ch_count; channel++) { 1856 denali_phy = dram->chan[channel].publ->denali_phy; 1857 clrsetbits_le32(&denali_phy[896], (0x3 << 8) | 1, 1 << 8); 1858 ret = data_training(&dram->chan[channel], channel, 1859 sdram_params, PI_FULL_TRAINING); 1860 if (ret) { 1861 debug("index1 training failed\n"); 1862 return ret; 1863 } 1864 } 1865 1866 return 0; 1867 } 1868 1869 u16 ddr_cfg_2_rbc[] = { 1870 /* 1871 * [6] highest bit col 1872 * [5:3] max row(14+n) 1873 * [2] insertion row 1874 * [1:0] col(9+n),col, data bus 32bit 1875 * 1876 * highbitcol, max_row, insertion_row, col 1877 */ 1878 ((0 << 6) | (2 << 3) | (0 << 2) | 0), /* 0 */ 1879 ((0 << 6) | (2 << 3) | (0 << 2) | 1), /* 1 */ 1880 ((0 << 6) | (1 << 3) | (0 << 2) | 2), /* 2 */ 1881 ((0 << 6) | (0 << 3) | (0 << 2) | 3), /* 3 */ 1882 ((0 << 6) | (2 << 3) | (1 << 2) | 1), /* 4 */ 1883 ((0 << 6) | (1 << 3) | (1 << 2) | 2), /* 5 */ 1884 ((1 << 6) | (0 << 3) | (0 << 2) | 2), /* 6 */ 1885 ((1 << 6) | (1 << 3) | (0 << 2) | 2), /* 7 */ 1886 }; 1887 1888 static u32 calculate_ddrconfig(struct rk3399_sdram_params *sdram_params, 1889 u32 channel) 1890 { 1891 unsigned int i; 1892 unsigned int cs0_row = sdram_params->ch[channel].cap_info.cs0_row; 1893 unsigned int col = sdram_params->ch[channel].cap_info.col; 1894 unsigned int bw = sdram_params->ch[channel].cap_info.bw; 1895 1896 col -= (bw == 2) ? 0 : 1; 1897 col -= 9; 1898 1899 for (i = 0; i < 4; i++) { 1900 if ((col == (ddr_cfg_2_rbc[i] & 0x3)) && 1901 (cs0_row <= (((ddr_cfg_2_rbc[i] >> 3) & 0x7) + 14))) 1902 break; 1903 } 1904 1905 if (i >= 4) 1906 i = -1; 1907 1908 return i; 1909 } 1910 1911 static unsigned char calculate_stride(struct rk3399_sdram_params *sdram_params) 1912 { 1913 unsigned int gstride_type; 1914 unsigned int channel; 1915 unsigned int chinfo = 0; 1916 unsigned int cap = 0; 1917 unsigned int stride = -1; 1918 unsigned int ch_cap[2] = {0, 0}; 1919 1920 gstride_type = STRIDE_256B; 1921 1922 for (channel = 0; channel < 2; channel++) { 1923 unsigned int cs0_cap = 0; 1924 unsigned int cs1_cap = 0; 1925 struct sdram_cap_info *cap_info = 1926 &sdram_params->ch[channel].cap_info; 1927 1928 if (cap_info->col == 0) 1929 continue; 1930 1931 cs0_cap = (1 << (cap_info->cs0_row + cap_info->col + 1932 cap_info->bk + cap_info->bw - 20)); 1933 if (cap_info->rank > 1) 1934 cs1_cap = cs0_cap >> (cap_info->cs0_row 1935 - cap_info->cs1_row); 1936 if (cap_info->row_3_4) { 1937 cs0_cap = cs0_cap * 3 / 4; 1938 cs1_cap = cs1_cap * 3 / 4; 1939 } 1940 ch_cap[channel] = cs0_cap + cs1_cap; 1941 chinfo |= 1 << channel; 1942 } 1943 1944 cap = ch_cap[0] + ch_cap[1]; 1945 if (sdram_params->base.num_channels == 1) { 1946 if (chinfo & 1) /* channel a only */ 1947 stride = 0x17; 1948 else /* channel b only */ 1949 stride = 0x18; 1950 } else {/* 2 channel */ 1951 if (ch_cap[0] == ch_cap[1]) { 1952 /* interleaved */ 1953 if (gstride_type == PART_STRIDE) { 1954 /* 1955 * first 64MB no interleaved other 256B interleaved 1956 * if 786M+768M.useful space from 0-1280MB and 1957 * 1536MB-1792MB 1958 * if 1.5G+1.5G(continuous).useful space from 0-2560MB 1959 * and 3072MB-3584MB 1960 */ 1961 stride = 0x1F; 1962 } else { 1963 switch (cap) { 1964 /* 512MB */ 1965 case 512: 1966 stride = 0; 1967 break; 1968 /* 1GB unstride or 256B stride*/ 1969 case 1024: 1970 stride = (gstride_type == UN_STRIDE) ? 1971 0x1 : 0x5; 1972 break; 1973 /* 1974 * 768MB + 768MB same as total 2GB memory 1975 * useful space: 0-768MB 1GB-1792MB 1976 */ 1977 case 1536: 1978 /* 2GB unstride or 256B or 512B stride */ 1979 case 2048: 1980 stride = (gstride_type == UN_STRIDE) ? 1981 0x2 : 1982 ((gstride_type == STRIDE_512B) ? 1983 0xA : 0x9); 1984 break; 1985 /* 1536MB + 1536MB */ 1986 case 3072: 1987 stride = (gstride_type == UN_STRIDE) ? 1988 0x3 : 1989 ((gstride_type == STRIDE_512B) ? 1990 0x12 : 0x11); 1991 break; 1992 /* 4GB unstride or 128B,256B,512B,4KB stride */ 1993 case 4096: 1994 stride = (gstride_type == UN_STRIDE) ? 1995 0x3 : (0xC + gstride_type); 1996 break; 1997 } 1998 } 1999 } 2000 if (ch_cap[0] == 2048 && ch_cap[1] == 1024) { 2001 /* 2GB + 1GB */ 2002 stride = (gstride_type == UN_STRIDE) ? 0x3 : 0x19; 2003 } 2004 /* 2005 * remain two channel capability not equal OR capability 2006 * power function of 2 2007 */ 2008 if (stride == (-1)) { 2009 switch ((ch_cap[0] > ch_cap[1]) ? 2010 ch_cap[0] : ch_cap[1]) { 2011 case 256: /* 256MB + 128MB */ 2012 stride = 0; 2013 break; 2014 case 512: /* 512MB + 256MB */ 2015 stride = 1; 2016 break; 2017 case 1024:/* 1GB + 128MB/256MB/384MB/512MB/768MB */ 2018 stride = 2; 2019 break; 2020 case 2048: /* 2GB + 128MB/256MB/384MB/512MB/768MB/1GB */ 2021 stride = 3; 2022 break; 2023 default: 2024 break; 2025 } 2026 } 2027 if (stride == (-1)) 2028 goto error; 2029 } 2030 switch (stride) { 2031 case 0xc: 2032 printf("128B stride\n"); 2033 break; 2034 case 5: 2035 case 9: 2036 case 0xd: 2037 case 0x11: 2038 case 0x19: 2039 printf("256B stride\n"); 2040 break; 2041 case 0xa: 2042 case 0xe: 2043 case 0x12: 2044 printf("512B stride\n"); 2045 break; 2046 case 0xf: 2047 printf("4K stride\n"); 2048 break; 2049 case 0x1f: 2050 printf("32MB + 256B stride\n"); 2051 break; 2052 default: 2053 printf("no stride\n"); 2054 } 2055 2056 return stride; 2057 error: 2058 printf("Cap not support!\n"); 2059 return (-1); 2060 } 2061 2062 static u32 get_ddr_stride(struct rk3399_pmusgrf_regs *pmusgrf) 2063 { 2064 u32 val; 2065 2066 val = (readl(&pmusgrf->soc_con4) >> 10) & 0x1F; 2067 2068 return val; 2069 } 2070 2071 static void set_ddr_stride(struct rk3399_pmusgrf_regs *pmusgrf, u32 stride) 2072 { 2073 rk_clrsetreg(&pmusgrf->soc_con4, 0x1f << 10, 2074 stride << 10); 2075 } 2076 2077 static void set_cap_relate_config(const struct chan_info *chan, 2078 struct rk3399_sdram_params *sdram_params, 2079 unsigned int channel) 2080 { 2081 u32 *denali_ctl = chan->pctl->denali_ctl; 2082 u32 tmp; 2083 struct sdram_msch_timings *noc_timing; 2084 2085 if (sdram_params->base.dramtype == LPDDR3) { 2086 tmp = (8 << sdram_params->ch[channel].cap_info.bw) / 2087 (8 << sdram_params->ch[channel].cap_info.dbw); 2088 /* memdata_ratio 2089 * 1 -> 0, 2 -> 1, 4 -> 2 2090 */ 2091 clrsetbits_le32(&denali_ctl[197], 0x7, 2092 (tmp >> 1)); 2093 clrsetbits_le32(&denali_ctl[198], 0x7 << 8, 2094 (tmp >> 1) << 8); 2095 } 2096 noc_timing = &sdram_params->ch[channel].noc_timings; 2097 /* 2098 * noc timing bw relate timing is 32 bit, and real bw is 16bit 2099 * actually noc reg is setting at function dram_all_config 2100 */ 2101 if (sdram_params->ch[channel].cap_info.bw == 16 && 2102 noc_timing->ddrmode.b.mwrsize == 2) { 2103 if (noc_timing->ddrmode.b.burstsize) 2104 noc_timing->ddrmode.b.burstsize -= 1; 2105 noc_timing->ddrmode.b.mwrsize -= 1; 2106 noc_timing->ddrtimingc0.b.burstpenalty *= 2; 2107 noc_timing->ddrtimingc0.b.wrtomwr *= 2; 2108 } 2109 } 2110 2111 static void clear_channel_params(struct rk3399_sdram_params *sdram_params, 2112 unsigned int channel) 2113 { 2114 sdram_params->ch[channel].cap_info.rank = 0; 2115 sdram_params->ch[channel].cap_info.col = 0; 2116 sdram_params->ch[channel].cap_info.bk = 0; 2117 sdram_params->ch[channel].cap_info.bw = 32; 2118 sdram_params->ch[channel].cap_info.dbw = 32; 2119 sdram_params->ch[channel].cap_info.row_3_4 = 0; 2120 sdram_params->ch[channel].cap_info.cs0_row = 0; 2121 sdram_params->ch[channel].cap_info.cs1_row = 0; 2122 sdram_params->ch[channel].cap_info.ddrconfig = 0; 2123 } 2124 2125 /* CS0,n=1 2126 * CS1,n=2 2127 * CS0 & CS1, n=3 2128 * cs0_cap: MB unit 2129 */ 2130 static void dram_set_cs(const struct chan_info *chan, u32 cs_map, u32 cs0_cap, 2131 unsigned char dramtype) 2132 { 2133 u32 *denali_ctl = chan->pctl->denali_ctl; 2134 u32 *denali_pi = chan->pi->denali_pi; 2135 struct msch_regs *ddr_msch_regs = chan->msch; 2136 2137 clrsetbits_le32(&denali_ctl[196], 0x3, cs_map); 2138 writel((cs0_cap / 32) | (((4096 - cs0_cap) / 32) << 8), 2139 &ddr_msch_regs->ddrsize); 2140 if (dramtype == LPDDR4) { 2141 if (cs_map == 1) 2142 cs_map = 0x5; 2143 else if (cs_map == 2) 2144 cs_map = 0xa; 2145 else 2146 cs_map = 0xF; 2147 } 2148 /*PI_41 PI_CS_MAP:RW:24:4*/ 2149 clrsetbits_le32(&denali_pi[41], 2150 0xf << 24, cs_map << 24); 2151 if (cs_map == 1 && dramtype == DDR3) 2152 writel(0x2EC7FFFF, &denali_pi[34]); 2153 } 2154 2155 static void dram_set_bw(const struct chan_info *chan, u32 bw) 2156 { 2157 u32 *denali_ctl = chan->pctl->denali_ctl; 2158 2159 if (bw == 2) 2160 clrbits_le32(&denali_ctl[196], 1 << 16); 2161 else 2162 setbits_le32(&denali_ctl[196], 1 << 16); 2163 } 2164 2165 static void dram_set_max_col(const struct chan_info *chan, u32 bw, u32 *pcol) 2166 { 2167 u32 *denali_ctl = chan->pctl->denali_ctl; 2168 struct msch_regs *ddr_msch_regs = chan->msch; 2169 u32 *denali_pi = chan->pi->denali_pi; 2170 u32 ddrconfig; 2171 2172 clrbits_le32(&denali_ctl[191], 0xf); 2173 clrsetbits_le32(&denali_ctl[190], 2174 (7 << 24), 2175 ((16 - ((bw == 2) ? 14 : 15)) << 24)); 2176 /*PI_199 PI_COL_DIFF:RW:0:4*/ 2177 clrbits_le32(&denali_pi[199], 0xf); 2178 /*PI_155 PI_ROW_DIFF:RW:24:3*/ 2179 clrsetbits_le32(&denali_pi[155], 2180 (7 << 24), 2181 ((16 - 12) << 24)); 2182 ddrconfig = (bw == 2) ? 3 : 2; 2183 writel(ddrconfig | (ddrconfig << 8), &ddr_msch_regs->ddrconf); 2184 /* set max cs0 size */ 2185 writel((4096 / 32) | ((0 / 32) << 8), 2186 &ddr_msch_regs->ddrsize); 2187 2188 *pcol = 12; 2189 } 2190 2191 static void dram_set_max_bank(const struct chan_info *chan, u32 bw, u32 *pbank, 2192 u32 *pcol) 2193 { 2194 u32 *denali_ctl = chan->pctl->denali_ctl; 2195 u32 *denali_pi = chan->pi->denali_pi; 2196 2197 clrbits_le32(&denali_ctl[191], 0xf); 2198 clrbits_le32(&denali_ctl[190], (3 << 16)); 2199 /*PI_199 PI_COL_DIFF:RW:0:4*/ 2200 clrbits_le32(&denali_pi[199], 0xf); 2201 /*PI_155 PI_BANK_DIFF:RW:16:2*/ 2202 clrbits_le32(&denali_pi[155], (3 << 16)); 2203 2204 *pbank = 3; 2205 *pcol = 12; 2206 } 2207 2208 static void dram_set_max_row(const struct chan_info *chan, u32 bw, u32 *prow, 2209 u32 *pbank, u32 *pcol) 2210 { 2211 u32 *denali_ctl = chan->pctl->denali_ctl; 2212 u32 *denali_pi = chan->pi->denali_pi; 2213 struct msch_regs *ddr_msch_regs = chan->msch; 2214 2215 clrsetbits_le32(&denali_ctl[191], 0xf, 12 - 10); 2216 clrbits_le32(&denali_ctl[190], 2217 (0x3 << 16) | (0x7 << 24)); 2218 /*PI_199 PI_COL_DIFF:RW:0:4*/ 2219 clrsetbits_le32(&denali_pi[199], 0xf, 12 - 10); 2220 /*PI_155 PI_ROW_DIFF:RW:24:3 PI_BANK_DIFF:RW:16:2*/ 2221 clrbits_le32(&denali_pi[155], 2222 (0x3 << 16) | (0x7 << 24)); 2223 writel(1 | (1 << 8), &ddr_msch_regs->ddrconf); 2224 /* set max cs0 size */ 2225 writel((4096 / 32) | ((0 / 32) << 8), 2226 &ddr_msch_regs->ddrsize); 2227 2228 *prow = 16; 2229 *pbank = 3; 2230 *pcol = (bw == 2) ? 10 : 11; 2231 } 2232 2233 static u64 dram_detect_cap(struct dram_info *dram, 2234 struct rk3399_sdram_params *sdram_params, 2235 unsigned char channel) 2236 { 2237 const struct chan_info *chan = &dram->chan[channel]; 2238 struct sdram_cap_info *cap_info = &sdram_params->ch[channel].cap_info; 2239 u32 bw; 2240 u32 col_tmp; 2241 u32 bk_tmp; 2242 u32 row_tmp; 2243 u32 cs0_cap; 2244 u32 training_flag; 2245 u32 ddrconfig; 2246 2247 /* detect bw */ 2248 bw = 2; 2249 if (sdram_params->base.dramtype != LPDDR4) { 2250 dram_set_bw(chan, bw); 2251 cap_info->bw = bw; 2252 if (data_training(chan, channel, sdram_params, 2253 PI_READ_GATE_TRAINING)) { 2254 bw = 1; 2255 dram_set_bw(chan, 1); 2256 cap_info->bw = bw; 2257 if (data_training(chan, channel, sdram_params, 2258 PI_READ_GATE_TRAINING)) { 2259 printf("16bit error!!!\n"); 2260 goto error; 2261 } 2262 } 2263 } 2264 /* 2265 * LPDDR3 CA training msut be trigger before other training. 2266 * DDR3 is not have CA training. 2267 */ 2268 if (sdram_params->base.dramtype == LPDDR3) 2269 training_flag = PI_WRITE_LEVELING; 2270 else 2271 training_flag = PI_FULL_TRAINING; 2272 2273 if (sdram_params->base.dramtype != LPDDR4) { 2274 if (data_training(chan, channel, sdram_params, training_flag)) { 2275 printf("full training error!!!\n"); 2276 goto error; 2277 } 2278 } 2279 2280 /* detect col */ 2281 dram_set_max_col(chan, bw, &col_tmp); 2282 if (sdram_detect_col(cap_info, col_tmp) != 0) 2283 goto error; 2284 2285 /* detect bank */ 2286 dram_set_max_bank(chan, bw, &bk_tmp, &col_tmp); 2287 sdram_detect_bank(cap_info, col_tmp, bk_tmp); 2288 2289 /* detect row */ 2290 dram_set_max_row(chan, bw, &row_tmp, &bk_tmp, &col_tmp); 2291 if (sdram_detect_row(cap_info, col_tmp, bk_tmp, row_tmp) != 0) 2292 goto error; 2293 2294 /* detect row_3_4 */ 2295 sdram_detect_row_3_4(cap_info, col_tmp, bk_tmp); 2296 2297 /* set ddrconfig */ 2298 cs0_cap = (1 << (cap_info->cs0_row + cap_info->col + cap_info->bk + 2299 cap_info->bw - 20)); 2300 if (cap_info->row_3_4) 2301 cs0_cap = cs0_cap * 3 / 4; 2302 2303 cap_info->cs1_row = cap_info->cs0_row; 2304 set_memory_map(chan, channel, sdram_params); 2305 ddrconfig = calculate_ddrconfig(sdram_params, channel); 2306 if (-1 == ddrconfig) 2307 goto error; 2308 set_ddrconfig(chan, sdram_params, channel, 2309 cap_info->ddrconfig); 2310 2311 /* detect cs1 row */ 2312 sdram_detect_cs1_row(cap_info, sdram_params->base.dramtype); 2313 2314 /* detect die bw */ 2315 sdram_detect_dbw(cap_info, sdram_params->base.dramtype); 2316 2317 return 0; 2318 error: 2319 return (-1); 2320 } 2321 2322 /* read mr_num mode register 2323 * input: rank = 1: cs0, rank = 2: cs1 2324 * mr_num: mode register number 2325 * output: buf 2326 */ 2327 static int read_mr(struct rk3399_ddr_pctl_regs *ddr_pctl_regs, u32 rank, 2328 u32 mr_num, u32 *buf) 2329 { 2330 s32 timeout = 100; 2331 2332 writel(((1 << 16) | 2333 (((rank == 2) ? 1 : 0) << 8) | 2334 mr_num) << 8, 2335 &ddr_pctl_regs->denali_ctl[118]); 2336 while (0 == (readl(&ddr_pctl_regs->denali_ctl[203]) & 2337 ((1 << 21) | (1 << 12)))) { 2338 udelay(1); 2339 if (timeout <= 0) 2340 goto error; 2341 timeout--; 2342 } 2343 if (!(readl(&ddr_pctl_regs->denali_ctl[203]) & (1 << 12))) { 2344 *buf = readl(&ddr_pctl_regs->denali_ctl[119]) & 0xFF; 2345 } else { 2346 printf("read mr error\n"); 2347 printf("MRR_ERROR_STATUS = 0x%x\n", 2348 readl(&ddr_pctl_regs->denali_ctl[17]) & 0x3); 2349 *buf = 0; 2350 } 2351 setbits_le32(&ddr_pctl_regs->denali_ctl[205], (1 << 21) | (1 << 12)); 2352 return 0; 2353 error: 2354 return (-1); 2355 } 2356 2357 static int read_mr_for_detect(struct dram_info *dram, u32 channel, u32 rank, 2358 struct rk3399_sdram_params *sdram_params) 2359 { 2360 u64 cs0_cap; 2361 u32 stride; 2362 u32 cs = 0, col = 0, bk = 0, bw = 0, row_3_4 = 0; 2363 u32 cs0_row = 0, cs1_row = 0, ddrconfig = 0; 2364 u32 mr5, mr12, mr14; 2365 struct chan_info *chan = 2366 &dram->chan[channel]; 2367 struct rk3399_ddr_pctl_regs *ddr_pctl_regs = chan->pctl; 2368 int ret = 0; 2369 u32 val; 2370 void __iomem *addr = NULL; 2371 2372 stride = get_ddr_stride(dram->pmusgrf); 2373 2374 if (sdram_params->ch[channel].cap_info.col == 0) { 2375 ret = -1; 2376 goto end; 2377 } 2378 2379 cs = sdram_params->ch[channel].cap_info.rank; 2380 col = sdram_params->ch[channel].cap_info.col; 2381 bk = sdram_params->ch[channel].cap_info.bk; 2382 bw = sdram_params->ch[channel].cap_info.bw; 2383 row_3_4 = sdram_params->ch[channel].cap_info.row_3_4; 2384 cs0_row = sdram_params->ch[channel].cap_info.cs0_row; 2385 cs1_row = sdram_params->ch[channel].cap_info.cs1_row; 2386 ddrconfig = sdram_params->ch[channel].cap_info.ddrconfig; 2387 2388 /* 2GB */ 2389 sdram_params->ch[channel].cap_info.rank = 2; 2390 sdram_params->ch[channel].cap_info.col = 10; 2391 sdram_params->ch[channel].cap_info.bk = 3; 2392 sdram_params->ch[channel].cap_info.bw = 2; 2393 sdram_params->ch[channel].cap_info.row_3_4 = 0; 2394 sdram_params->ch[channel].cap_info.cs0_row = 15; 2395 sdram_params->ch[channel].cap_info.cs1_row = 15; 2396 sdram_params->ch[channel].cap_info.ddrconfig = 1; 2397 2398 set_memory_map(chan, channel, sdram_params); 2399 sdram_params->ch[channel].cap_info.ddrconfig = 2400 calculate_ddrconfig(sdram_params, channel); 2401 set_ddrconfig(chan, sdram_params, channel, 2402 sdram_params->ch[channel].cap_info.ddrconfig); 2403 set_cap_relate_config(chan, sdram_params, channel); 2404 2405 cs0_cap = (1 << (sdram_params->ch[channel].cap_info.bw 2406 + sdram_params->ch[channel].cap_info.col 2407 + sdram_params->ch[channel].cap_info.bk 2408 + sdram_params->ch[channel].cap_info.cs0_row)); 2409 2410 if (sdram_params->ch[channel].cap_info.row_3_4) 2411 cs0_cap = cs0_cap * 3 / 4; 2412 2413 if (channel == 0) 2414 set_ddr_stride(dram->pmusgrf, 0x17); 2415 else 2416 set_ddr_stride(dram->pmusgrf, 0x18); 2417 2418 /* !will soiled DRAM space here! 2419 * read and write data to DRAM, avoid be optimized by compiler. 2420 */ 2421 if (rank == 1) 2422 addr = (void __iomem *)0x100; 2423 else if (rank == 2) 2424 addr = (void __iomem *)(cs0_cap + 0x100); 2425 2426 val = readl(addr); 2427 writel(val + 1, addr); 2428 2429 read_mr(ddr_pctl_regs, rank, 5, &mr5); 2430 read_mr(ddr_pctl_regs, rank, 12, &mr12); 2431 read_mr(ddr_pctl_regs, rank, 14, &mr14); 2432 2433 if (mr5 == 0 || mr12 != 0x4d || mr14 != 0x4d) { 2434 ret = -1; 2435 goto end; 2436 } 2437 end: 2438 sdram_params->ch[channel].cap_info.rank = cs; 2439 sdram_params->ch[channel].cap_info.col = col; 2440 sdram_params->ch[channel].cap_info.bk = bk; 2441 sdram_params->ch[channel].cap_info.bw = bw; 2442 sdram_params->ch[channel].cap_info.row_3_4 = row_3_4; 2443 sdram_params->ch[channel].cap_info.cs0_row = cs0_row; 2444 sdram_params->ch[channel].cap_info.cs1_row = cs1_row; 2445 sdram_params->ch[channel].cap_info.ddrconfig = ddrconfig; 2446 2447 set_ddr_stride(dram->pmusgrf, stride); 2448 return ret; 2449 } 2450 2451 static u32 get_phy_fn(struct rk3399_sdram_params *sdram_params, u32 ctl_fn) 2452 { 2453 u32 lp4_phy_fn[] = {1, 0, 0xb}; 2454 2455 if (sdram_params->base.dramtype == LPDDR4) 2456 return lp4_phy_fn[ctl_fn]; 2457 else 2458 return ctl_fn; 2459 } 2460 2461 static u32 get_ctl_fn(struct rk3399_sdram_params *sdram_params, u32 phy_fn) 2462 { 2463 u32 lp4_ctl_fn[] = {1, 0, 2}; 2464 2465 if (sdram_params->base.dramtype == LPDDR4) 2466 return lp4_ctl_fn[phy_fn]; 2467 else 2468 return phy_fn; 2469 } 2470 2471 static void dram_copy_phy_fn(struct dram_info *dram, 2472 struct rk3399_sdram_params *sdram_params, u32 fn, 2473 struct rk3399_sdram_params *f1_sdram_params, 2474 u32 channel) 2475 { 2476 u32 *denali_ctl; 2477 u32 *denali_phy; 2478 u32 *denali_phy_params; 2479 u32 speed = 0; 2480 u32 mr5; 2481 u32 ctl_fn; 2482 2483 denali_ctl = dram->chan[channel].pctl->denali_ctl; 2484 denali_phy = dram->chan[channel].publ->denali_phy; 2485 denali_phy_params = f1_sdram_params->phy_regs.denali_phy; 2486 2487 /* switch index */ 2488 clrsetbits_le32(&denali_phy_params[896], 0x3 << 8, 2489 fn << 8); 2490 writel(denali_phy_params[896], &denali_phy[896]); 2491 2492 /* phy_pll_ctrl_ca, phy_pll_ctrl */ 2493 writel(denali_phy_params[911], &denali_phy[911]); 2494 /* phy_low_freq_sel */ 2495 clrsetbits_le32(&denali_phy[913], 0x1, 2496 denali_phy_params[913] & 0x1); 2497 /* PHY_GRP_SLAVE_DELAY_X, phy_cslvl_dly_step */ 2498 writel(denali_phy_params[916], &denali_phy[916]); 2499 writel(denali_phy_params[917], &denali_phy[917]); 2500 writel(denali_phy_params[918], &denali_phy[918]); 2501 /* phy_adrZ_sw_wraddr_shift_X */ 2502 writel(denali_phy_params[512], &denali_phy[512]); 2503 clrsetbits_le32(&denali_phy[513], 0xFFFF, 2504 denali_phy_params[513] & 0xFFFF); 2505 writel(denali_phy_params[640], &denali_phy[640]); 2506 clrsetbits_le32(&denali_phy[641], 0xFFFF, 2507 denali_phy_params[641] & 0xFFFF); 2508 writel(denali_phy_params[768], &denali_phy[768]); 2509 clrsetbits_le32(&denali_phy[769], 0xFFFF, 2510 denali_phy_params[769] & 0xFFFF); 2511 2512 writel(denali_phy_params[544], &denali_phy[544]); 2513 writel(denali_phy_params[545], &denali_phy[545]); 2514 writel(denali_phy_params[546], &denali_phy[546]); 2515 writel(denali_phy_params[547], &denali_phy[547]); 2516 2517 writel(denali_phy_params[672], &denali_phy[672]); 2518 writel(denali_phy_params[673], &denali_phy[673]); 2519 writel(denali_phy_params[674], &denali_phy[674]); 2520 writel(denali_phy_params[675], &denali_phy[675]); 2521 2522 writel(denali_phy_params[800], &denali_phy[800]); 2523 writel(denali_phy_params[801], &denali_phy[801]); 2524 writel(denali_phy_params[802], &denali_phy[802]); 2525 writel(denali_phy_params[803], &denali_phy[803]); 2526 2527 /* 2528 * phy_adr_master_delay_start_X 2529 * phy_adr_master_delay_step_X 2530 * phy_adr_master_delay_wait_X 2531 */ 2532 writel(denali_phy_params[548], &denali_phy[548]); 2533 writel(denali_phy_params[676], &denali_phy[676]); 2534 writel(denali_phy_params[804], &denali_phy[804]); 2535 2536 /* phy_adr_calvl_dly_step_X */ 2537 writel(denali_phy_params[549], &denali_phy[549]); 2538 writel(denali_phy_params[677], &denali_phy[677]); 2539 writel(denali_phy_params[805], &denali_phy[805]); 2540 2541 /* 2542 * phy_clk_wrdm_slave_delay_X 2543 * phy_clk_wrdqZ_slave_delay_X 2544 * phy_clk_wrdqs_slave_delay_X 2545 */ 2546 sdram_copy_to_reg((u32 *)&denali_phy[59], 2547 (u32 *)&denali_phy_params[59], 2548 (63 - 58) * 4); 2549 sdram_copy_to_reg((u32 *)&denali_phy[187], 2550 (u32 *)&denali_phy_params[187], 2551 (191 - 186) * 4); 2552 sdram_copy_to_reg((u32 *)&denali_phy[315], 2553 (u32 *)&denali_phy_params[315], 2554 (319 - 314) * 4); 2555 sdram_copy_to_reg((u32 *)&denali_phy[443], 2556 (u32 *)&denali_phy_params[443], 2557 (447 - 442) * 4); 2558 2559 /* 2560 * phy_dqs_tsel_wr_timing_X 8bits DENALI_PHY_84/212/340/468 offset_8 2561 * dqs_tsel_wr_end[7:4] add Half cycle 2562 * phy_dq_tsel_wr_timing_X 8bits DENALI_PHY_83/211/339/467 offset_8 2563 * dq_tsel_wr_end[7:4] add Half cycle 2564 */ 2565 writel(denali_phy_params[83] + (0x10 << 16), &denali_phy[83]); 2566 writel(denali_phy_params[84] + (0x10 << 8), &denali_phy[84]); 2567 writel(denali_phy_params[85], &denali_phy[85]); 2568 2569 writel(denali_phy_params[211] + (0x10 << 16), &denali_phy[211]); 2570 writel(denali_phy_params[212] + (0x10 << 8), &denali_phy[212]); 2571 writel(denali_phy_params[213], &denali_phy[213]); 2572 2573 writel(denali_phy_params[339] + (0x10 << 16), &denali_phy[339]); 2574 writel(denali_phy_params[340] + (0x10 << 8), &denali_phy[340]); 2575 writel(denali_phy_params[341], &denali_phy[341]); 2576 2577 writel(denali_phy_params[467] + (0x10 << 16), &denali_phy[467]); 2578 writel(denali_phy_params[468] + (0x10 << 8), &denali_phy[468]); 2579 writel(denali_phy_params[469], &denali_phy[469]); 2580 2581 /* 2582 * phy_gtlvl_resp_wait_cnt_X 2583 * phy_gtlvl_dly_step_X 2584 * phy_wrlvl_resp_wait_cnt_X 2585 * phy_gtlvl_final_step_X 2586 * phy_gtlvl_back_step_X 2587 * phy_rdlvl_dly_step_X 2588 * 2589 * phy_master_delay_step_X 2590 * phy_master_delay_wait_X 2591 * phy_wrlvl_dly_step_X 2592 * phy_rptr_update_X 2593 * phy_wdqlvl_dly_step_X 2594 */ 2595 writel(denali_phy_params[87], &denali_phy[87]); 2596 writel(denali_phy_params[88], &denali_phy[88]); 2597 writel(denali_phy_params[89], &denali_phy[89]); 2598 writel(denali_phy_params[90], &denali_phy[90]); 2599 2600 writel(denali_phy_params[215], &denali_phy[215]); 2601 writel(denali_phy_params[216], &denali_phy[216]); 2602 writel(denali_phy_params[217], &denali_phy[217]); 2603 writel(denali_phy_params[218], &denali_phy[218]); 2604 2605 writel(denali_phy_params[343], &denali_phy[343]); 2606 writel(denali_phy_params[344], &denali_phy[344]); 2607 writel(denali_phy_params[345], &denali_phy[345]); 2608 writel(denali_phy_params[346], &denali_phy[346]); 2609 2610 writel(denali_phy_params[471], &denali_phy[471]); 2611 writel(denali_phy_params[472], &denali_phy[472]); 2612 writel(denali_phy_params[473], &denali_phy[473]); 2613 writel(denali_phy_params[474], &denali_phy[474]); 2614 2615 /* 2616 * phy_gtlvl_lat_adj_start_X 2617 * phy_gtlvl_rddqs_slv_dly_start_X 2618 * phy_rdlvl_rddqs_dq_slv_dly_start_X 2619 * phy_wdqlvl_dqdm_slv_dly_start_X 2620 */ 2621 writel(denali_phy_params[80], &denali_phy[80]); 2622 writel(denali_phy_params[81], &denali_phy[81]); 2623 2624 writel(denali_phy_params[208], &denali_phy[208]); 2625 writel(denali_phy_params[209], &denali_phy[209]); 2626 2627 writel(denali_phy_params[336], &denali_phy[336]); 2628 writel(denali_phy_params[337], &denali_phy[337]); 2629 2630 writel(denali_phy_params[464], &denali_phy[464]); 2631 writel(denali_phy_params[465], &denali_phy[465]); 2632 2633 /* 2634 * phy_master_delay_start_X 2635 * phy_sw_master_mode_X 2636 * phy_rddata_en_tsel_dly_X 2637 */ 2638 writel(denali_phy_params[86], &denali_phy[86]); 2639 writel(denali_phy_params[214], &denali_phy[214]); 2640 writel(denali_phy_params[342], &denali_phy[342]); 2641 writel(denali_phy_params[470], &denali_phy[470]); 2642 2643 /* 2644 * phy_rddqZ_slave_delay_X 2645 * phy_rddqs_dqZ_fall_slave_delay_X 2646 * phy_rddqs_dqZ_rise_slave_delay_X 2647 * phy_rddqs_dm_fall_slave_delay_X 2648 * phy_rddqs_dm_rise_slave_delay_X 2649 * phy_rddqs_gate_slave_delay_X 2650 * phy_wrlvl_delay_early_threshold_X 2651 * phy_write_path_lat_add_X 2652 * phy_rddqs_latency_adjust_X 2653 * phy_wrlvl_delay_period_threshold_X 2654 * phy_wrlvl_early_force_zero_X 2655 */ 2656 sdram_copy_to_reg((u32 *)&denali_phy[64], 2657 (u32 *)&denali_phy_params[64], 2658 (67 - 63) * 4); 2659 clrsetbits_le32(&denali_phy[68], 0xFFFFFC00, 2660 denali_phy_params[68] & 0xFFFFFC00); 2661 sdram_copy_to_reg((u32 *)&denali_phy[69], 2662 (u32 *)&denali_phy_params[69], 2663 (79 - 68) * 4); 2664 2665 sdram_copy_to_reg((u32 *)&denali_phy[192], 2666 (u32 *)&denali_phy_params[192], 2667 (195 - 191) * 4); 2668 clrsetbits_le32(&denali_phy[196], 0xFFFFFC00, 2669 denali_phy_params[196] & 0xFFFFFC00); 2670 sdram_copy_to_reg((u32 *)&denali_phy[197], 2671 (u32 *)&denali_phy_params[197], 2672 (207 - 196) * 4); 2673 2674 sdram_copy_to_reg((u32 *)&denali_phy[320], 2675 (u32 *)&denali_phy_params[320], 2676 (323 - 319) * 4); 2677 clrsetbits_le32(&denali_phy[324], 0xFFFFFC00, 2678 denali_phy_params[324] & 0xFFFFFC00); 2679 sdram_copy_to_reg((u32 *)&denali_phy[325], 2680 (u32 *)&denali_phy_params[325], 2681 (335 - 324) * 4); 2682 2683 sdram_copy_to_reg((u32 *)&denali_phy[448], 2684 (u32 *)&denali_phy_params[448], 2685 (451 - 447) * 4); 2686 clrsetbits_le32(&denali_phy[452], 0xFFFFFC00, 2687 denali_phy_params[452] & 0xFFFFFC00); 2688 sdram_copy_to_reg((u32 *)&denali_phy[453], 2689 (u32 *)&denali_phy_params[453], 2690 (463 - 452) * 4); 2691 2692 /* phy_two_cyc_preamble_X */ 2693 clrsetbits_le32(&denali_phy[7], 0x3 << 24, 2694 denali_phy_params[7] & (0x3 << 24)); 2695 clrsetbits_le32(&denali_phy[135], 0x3 << 24, 2696 denali_phy_params[135] & (0x3 << 24)); 2697 clrsetbits_le32(&denali_phy[263], 0x3 << 24, 2698 denali_phy_params[263] & (0x3 << 24)); 2699 clrsetbits_le32(&denali_phy[391], 0x3 << 24, 2700 denali_phy_params[391] & (0x3 << 24)); 2701 2702 /* speed */ 2703 if (f1_sdram_params->base.ddr_freq < 400 * MHz) 2704 speed = 0x0; 2705 else if (f1_sdram_params->base.ddr_freq < 800 * MHz) 2706 speed = 0x1; 2707 else if (f1_sdram_params->base.ddr_freq < 1200 * MHz) 2708 speed = 0x2; 2709 2710 /* PHY_924 PHY_PAD_FDBK_DRIVE */ 2711 clrsetbits_le32(&denali_phy[924], 2712 0x3 << 21, speed << 21); 2713 /* PHY_926 PHY_PAD_DATA_DRIVE */ 2714 clrsetbits_le32(&denali_phy[926], 2715 0x3 << 9, speed << 9); 2716 /* PHY_927 PHY_PAD_DQS_DRIVE */ 2717 clrsetbits_le32(&denali_phy[927], 2718 0x3 << 9, speed << 9); 2719 /* PHY_928 PHY_PAD_ADDR_DRIVE */ 2720 clrsetbits_le32(&denali_phy[928], 2721 0x3 << 17, speed << 17); 2722 /* PHY_929 PHY_PAD_CLK_DRIVE */ 2723 clrsetbits_le32(&denali_phy[929], 2724 0x3 << 17, speed << 17); 2725 /* PHY_935 PHY_PAD_CKE_DRIVE */ 2726 clrsetbits_le32(&denali_phy[935], 2727 0x3 << 17, speed << 17); 2728 /* PHY_937 PHY_PAD_RST_DRIVE */ 2729 clrsetbits_le32(&denali_phy[937], 2730 0x3 << 17, speed << 17); 2731 /* PHY_939 PHY_PAD_CS_DRIVE */ 2732 clrsetbits_le32(&denali_phy[939], 2733 0x3 << 17, speed << 17); 2734 2735 if (f1_sdram_params->base.dramtype == LPDDR4) { 2736 read_mr(dram->chan[channel].pctl, 1, 5, &mr5); 2737 set_ds_odt(&dram->chan[channel], f1_sdram_params, 1, 0, mr5); 2738 set_ds_odt(&dram->chan[channel], f1_sdram_params, 1, 1, mr5); 2739 2740 ctl_fn = get_ctl_fn(f1_sdram_params, fn); 2741 set_lp4_dq_odt(&dram->chan[channel], f1_sdram_params, 2742 ctl_fn, 1, 1, 0, mr5); 2743 set_lp4_ca_odt(&dram->chan[channel], f1_sdram_params, 2744 ctl_fn, 1, 1, 0, mr5); 2745 set_lp4_MR3(&dram->chan[channel], f1_sdram_params, 2746 ctl_fn, 1, 0, mr5); 2747 set_lp4_MR12(&dram->chan[channel], f1_sdram_params, 2748 ctl_fn, 1, 0, mr5); 2749 set_lp4_MR14(&dram->chan[channel], f1_sdram_params, 2750 ctl_fn, 1, 0, mr5); 2751 2752 set_lp4_dq_odt(&dram->chan[channel], f1_sdram_params, 2753 ctl_fn, 1, 1, 1, mr5); 2754 set_lp4_ca_odt(&dram->chan[channel], f1_sdram_params, 2755 ctl_fn, 1, 1, 1, mr5); 2756 set_lp4_MR3(&dram->chan[channel], f1_sdram_params, 2757 ctl_fn, 1, 1, mr5); 2758 set_lp4_MR12(&dram->chan[channel], f1_sdram_params, 2759 ctl_fn, 1, 1, mr5); 2760 set_lp4_MR14(&dram->chan[channel], f1_sdram_params, 2761 ctl_fn, 1, 1, mr5); 2762 2763 /* 2764 * if phy_sw_master_mode_X not bypass mode, 2765 * clear PHY_SLICE_PWR_RDC_DISABLE. 2766 * NOTE: need use f1_sdram_params, not ddr_publ_regs 2767 */ 2768 if (!((denali_phy_params[86] >> 8) 2769 & (1 << 2))) { 2770 clrbits_le32(&denali_phy[10], 1 << 16); 2771 clrbits_le32(&denali_phy[138], 1 << 16); 2772 clrbits_le32(&denali_phy[266], 1 << 16); 2773 clrbits_le32(&denali_phy[394], 1 << 16); 2774 } 2775 2776 /* 2777 * when PHY_PER_CS_TRAINING_EN=1, W2W_DIFFCS_DLY_Fx can't 2778 * smaller than 8 2779 * NOTE: need use f1_sdram_params, not ddr_publ_regs 2780 */ 2781 if ((denali_phy_params[84] >> 16) & 1) { 2782 if (((readl(&denali_ctl[217 + ctl_fn]) >> 2783 16) & 0x1f) < 8) 2784 clrsetbits_le32(&denali_ctl[217 + ctl_fn], 2785 0x1f << 16, 2786 8 << 16); 2787 } 2788 } 2789 } 2790 2791 static void dram_set_phy_fn(struct dram_info *dram, 2792 struct rk3399_sdram_params *sdram_params, u32 fn, 2793 struct rk3399_sdram_params *f1_sdram_params) 2794 { 2795 u32 channel; 2796 2797 for (channel = 0; channel < 2; channel++) 2798 dram_copy_phy_fn(dram, sdram_params, fn, f1_sdram_params, 2799 channel); 2800 } 2801 2802 static int dram_set_rate(struct dram_info *dram, 2803 struct rk3399_sdram_params *sdram_params, 2804 u32 fn, u32 hz) 2805 { 2806 u32 channel; 2807 int ret_clk, ret[2]; 2808 2809 /* cci idle req stall */ 2810 writel(0x70007, &dram->grf->soc_con0); 2811 /* enable all clk */ 2812 setbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7)); 2813 /* idle */ 2814 setbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18)); 2815 while ((readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18)) 2816 != (0x3 << 18)) 2817 ; 2818 2819 /* change freq */ 2820 writel((((0x3 << 4) | (1 << 2) | 1) << 16) | 2821 (fn << 4) | (1 << 2) | 1, &dram->cic->cic_ctrl0); 2822 while (!(readl(&dram->cic->cic_status0) & (1 << 2))) 2823 ; 2824 2825 ret_clk = clk_set_rate(&dram->ddr_clk, hz); 2826 if (ret_clk < 0) { 2827 printf("%s clk set failed %d\n", __func__, ret_clk); 2828 return ret_clk; 2829 } 2830 2831 writel(0x20002, &dram->cic->cic_ctrl0); 2832 while (!(readl(&dram->cic->cic_status0) & (1 << 0))) 2833 ; 2834 2835 /* deidle */ 2836 clrbits_le32(&dram->pmu->pmu_bus_idle_req, (0x3 << 18)); 2837 while (readl(&dram->pmu->pmu_bus_idle_st) & (0x3 << 18)) 2838 ; 2839 2840 /* clear enable all clk */ 2841 clrbits_le32(&dram->pmu->pmu_noc_auto_ena, (0x3 << 7)); 2842 2843 /* LPDDR4 f2 can not do training, all training will fail */ 2844 if (!(sdram_params->base.dramtype == LPDDR4 && fn == 2)) { 2845 for (channel = 0; channel < 2; channel++) { 2846 if (!(sdram_params->ch[channel].cap_info.col)) 2847 continue; 2848 ret[channel] = data_training(&dram->chan[channel], 2849 channel, sdram_params, 2850 PI_FULL_TRAINING); 2851 } 2852 for (channel = 0; channel < 2; channel++) { 2853 if (!(sdram_params->ch[channel].cap_info.col)) 2854 continue; 2855 if (ret[channel]) 2856 printf("channel %d training failed!\n", 2857 channel); 2858 else 2859 printf("channel %d training pass\n", channel); 2860 } 2861 } 2862 2863 return 0; 2864 } 2865 2866 static struct rk3399_sdram_params *g_sdram_params; 2867 static void set_rate0(struct dram_info *dram) 2868 { 2869 u32 ctl_fn; 2870 u32 phy_fn; 2871 2872 ctl_fn = 0; 2873 phy_fn = get_phy_fn(g_sdram_params, ctl_fn); 2874 dram_set_phy_fn(dram, g_sdram_params, phy_fn, &dfs_configs[ctl_fn]); 2875 dram_set_rate(dram, g_sdram_params, ctl_fn, 2876 dfs_configs[ctl_fn].base.ddr_freq); 2877 printf("change freq to %d MHz %d, %d\n", 2878 dfs_configs[ctl_fn].base.ddr_freq / MHZ, ctl_fn, phy_fn); 2879 } 2880 2881 static void set_rate1(struct dram_info *dram) 2882 { 2883 u32 ctl_fn; 2884 u32 phy_fn; 2885 2886 ctl_fn = 1; 2887 phy_fn = get_phy_fn(g_sdram_params, ctl_fn); 2888 dram_set_phy_fn(dram, g_sdram_params, phy_fn, &dfs_configs[ctl_fn]); 2889 dram_set_rate(dram, g_sdram_params, ctl_fn, 2890 dfs_configs[ctl_fn].base.ddr_freq); 2891 printf("change freq to %d MHz %d, %d\n", 2892 dfs_configs[ctl_fn].base.ddr_freq / MHZ, ctl_fn, phy_fn); 2893 } 2894 2895 static int sdram_init(struct dram_info *dram, 2896 struct rk3399_sdram_params *sdram_params) 2897 { 2898 unsigned char dramtype = sdram_params->base.dramtype; 2899 unsigned int ddr_freq = sdram_params->base.ddr_freq; 2900 int channel; 2901 u32 rank; 2902 int ch; 2903 u32 tmp; 2904 u32 training_flag; 2905 2906 debug("Starting SDRAM initialization...\n"); 2907 2908 if ((dramtype == DDR3 && ddr_freq > 933) || 2909 (dramtype == LPDDR3 && ddr_freq > 933) || 2910 (dramtype == LPDDR4 && ddr_freq > 800)) { 2911 debug("SDRAM frequency is to high!"); 2912 return -E2BIG; 2913 } 2914 2915 /* detect rank */ 2916 for (ch = 0; ch < 2; ch++) { 2917 sdram_params->ch[ch].cap_info.rank = 2; 2918 for (rank = 2; rank != 0; rank--) { 2919 for (channel = 0; channel < 2; channel++) { 2920 const struct chan_info *chan = 2921 &dram->chan[channel]; 2922 struct rk3399_cru *cru = dram->cru; 2923 struct rk3399_ddr_publ_regs *publ = chan->publ; 2924 2925 phy_pctrl_reset(cru, channel); 2926 phy_dll_bypass_set(publ, ddr_freq); 2927 pctl_cfg(chan, channel, sdram_params); 2928 } 2929 2930 /* start to trigger initialization */ 2931 pctl_start(dram, sdram_params, 3); 2932 2933 /* LPDDR2/LPDDR3 need to wait DAI complete, max 10us */ 2934 if (dramtype == LPDDR3) 2935 udelay(10); 2936 2937 tmp = (rank == 2) ? 3 : 1; 2938 dram_set_cs(&dram->chan[ch], tmp, 2048, 2939 sdram_params->base.dramtype); 2940 sdram_params->ch[ch].cap_info.rank = rank; 2941 if (sdram_params->base.dramtype == LPDDR4) { 2942 /* two rank, then read_mr(cs1) 2943 * one rank, then read_mr(cs0) 2944 */ 2945 if (!read_mr_for_detect(dram, ch, rank, 2946 sdram_params)) 2947 break; 2948 } else { 2949 /* 2950 * LPDDR3 CA training msut be trigger before 2951 * other training. 2952 * DDR3 is not have CA training. 2953 * LPDDR4 need confirm here! 2954 */ 2955 if (sdram_params->base.dramtype == LPDDR3) 2956 training_flag = PI_CA_TRAINING | 2957 PI_READ_GATE_TRAINING; 2958 else 2959 training_flag = PI_READ_GATE_TRAINING; 2960 if (!(data_training(&dram->chan[ch], ch, 2961 sdram_params, 2962 training_flag))) 2963 break; 2964 } 2965 } 2966 sdram_params->ch[ch].cap_info.rank = rank; 2967 } 2968 2969 sdram_params->base.num_channels = 0; 2970 for (channel = 0; channel < 2; channel++) { 2971 const struct chan_info *chan = &dram->chan[channel]; 2972 struct sdram_cap_info *cap_info = 2973 &sdram_params->ch[channel].cap_info; 2974 2975 if (cap_info->rank == 0) { 2976 clear_channel_params(sdram_params, 1); 2977 continue; 2978 } else { 2979 sdram_params->base.num_channels++; 2980 } 2981 2982 printf("Channel "); 2983 printf(channel ? "1: " : "0: "); 2984 2985 if (channel == 0) 2986 set_ddr_stride(dram->pmusgrf, 0x17); 2987 else 2988 set_ddr_stride(dram->pmusgrf, 0x18); 2989 2990 if (dram_detect_cap(dram, sdram_params, channel)) { 2991 printf("Cap error!\n"); 2992 continue; 2993 } 2994 2995 sdram_print_ddr_info(cap_info, &sdram_params->base, 0); 2996 set_memory_map(chan, channel, sdram_params); 2997 cap_info->ddrconfig = 2998 calculate_ddrconfig(sdram_params, channel); 2999 if (-1 == cap_info->ddrconfig) { 3000 printf("no ddrconfig find, Cap not support!\n"); 3001 continue; 3002 } 3003 set_ddrconfig(chan, sdram_params, channel, cap_info->ddrconfig); 3004 set_cap_relate_config(chan, sdram_params, channel); 3005 } 3006 3007 if (sdram_params->base.num_channels == 0) { 3008 sdram_print_dram_type(sdram_params->base.dramtype); 3009 printf(" %dMHz\n", sdram_params->base.ddr_freq); 3010 return -1; 3011 } 3012 3013 sdram_params->base.stride = calculate_stride(sdram_params); 3014 dram_all_config(dram, sdram_params); 3015 3016 if (sdram_params->base.dramtype != LPDDR4) 3017 switch_to_phy_index1(dram, sdram_params); 3018 3019 if (sdram_params->base.dramtype == LPDDR4) { 3020 g_sdram_params = sdram_params; 3021 set_rate0(dram); 3022 set_rate1(dram); 3023 } 3024 3025 debug("Finish SDRAM initialization...\n"); 3026 return 0; 3027 } 3028 3029 static int rk3399_dmc_ofdata_to_platdata(struct udevice *dev) 3030 { 3031 #if !CONFIG_IS_ENABLED(OF_PLATDATA) 3032 struct rockchip_dmc_plat *plat = dev_get_platdata(dev); 3033 int ret; 3034 3035 ret = dev_read_u32_array(dev, "rockchip,sdram-params", 3036 (u32 *)&plat->sdram_params, 3037 sizeof(plat->sdram_params) / sizeof(u32)); 3038 if (ret) { 3039 printf("%s: Cannot read rockchip,sdram-params %d\n", 3040 __func__, ret); 3041 return ret; 3042 } 3043 ret = regmap_init_mem(dev, &plat->map); 3044 if (ret) 3045 printf("%s: regmap failed %d\n", __func__, ret); 3046 3047 #endif 3048 return 0; 3049 } 3050 3051 #if CONFIG_IS_ENABLED(OF_PLATDATA) 3052 static int conv_of_platdata(struct udevice *dev) 3053 { 3054 struct rockchip_dmc_plat *plat = dev_get_platdata(dev); 3055 struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat; 3056 int ret; 3057 3058 ret = regmap_init_mem_platdata(dev, dtplat->reg, 3059 ARRAY_SIZE(dtplat->reg) / 2, 3060 &plat->map); 3061 if (ret) 3062 return ret; 3063 3064 return 0; 3065 } 3066 #endif 3067 3068 static int rk3399_dmc_init(struct udevice *dev) 3069 { 3070 struct dram_info *priv = dev_get_priv(dev); 3071 struct rockchip_dmc_plat *plat = dev_get_platdata(dev); 3072 int ret; 3073 #if !CONFIG_IS_ENABLED(OF_PLATDATA) 3074 struct rk3399_sdram_params *params = &plat->sdram_params; 3075 #else 3076 struct dtd_rockchip_rk3399_dmc *dtplat = &plat->dtplat; 3077 struct rk3399_sdram_params *params = 3078 (void *)dtplat->rockchip_sdram_params; 3079 3080 ret = conv_of_platdata(dev); 3081 if (ret) 3082 return ret; 3083 #endif 3084 3085 priv->cic = syscon_get_first_range(ROCKCHIP_SYSCON_CIC); 3086 priv->grf = syscon_get_first_range(ROCKCHIP_SYSCON_GRF); 3087 priv->pmu = syscon_get_first_range(ROCKCHIP_SYSCON_PMU); 3088 priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF); 3089 priv->pmusgrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUSGRF); 3090 priv->pmucru = rockchip_get_pmucru(); 3091 priv->cru = rockchip_get_cru(); 3092 priv->chan[0].pctl = regmap_get_range(plat->map, 0); 3093 priv->chan[0].pi = regmap_get_range(plat->map, 1); 3094 priv->chan[0].publ = regmap_get_range(plat->map, 2); 3095 priv->chan[0].msch = regmap_get_range(plat->map, 3); 3096 priv->chan[1].pctl = regmap_get_range(plat->map, 4); 3097 priv->chan[1].pi = regmap_get_range(plat->map, 5); 3098 priv->chan[1].publ = regmap_get_range(plat->map, 6); 3099 priv->chan[1].msch = regmap_get_range(plat->map, 7); 3100 3101 debug("con reg %p %p %p %p %p %p %p %p\n", 3102 priv->chan[0].pctl, priv->chan[0].pi, 3103 priv->chan[0].publ, priv->chan[0].msch, 3104 priv->chan[1].pctl, priv->chan[1].pi, 3105 priv->chan[1].publ, priv->chan[1].msch); 3106 debug("cru %p, cic %p, grf %p, sgrf %p, pmucru %p, pmu %p\n", priv->cru, 3107 priv->cic, priv->pmugrf, priv->pmusgrf, priv->pmucru, priv->pmu); 3108 #if CONFIG_IS_ENABLED(OF_PLATDATA) 3109 ret = clk_get_by_index_platdata(dev, 0, dtplat->clocks, &priv->ddr_clk); 3110 #else 3111 ret = clk_get_by_index(dev, 0, &priv->ddr_clk); 3112 #endif 3113 if (ret) { 3114 printf("%s clk get failed %d\n", __func__, ret); 3115 return ret; 3116 } 3117 ret = clk_set_rate(&priv->ddr_clk, params->base.ddr_freq * MHz); 3118 if (ret < 0) { 3119 printf("%s clk set failed %d\n", __func__, ret); 3120 return ret; 3121 } 3122 ret = sdram_init(priv, params); 3123 if (ret < 0) { 3124 printf("%s DRAM init failed %d\n", __func__, ret); 3125 return ret; 3126 } 3127 3128 return 0; 3129 } 3130 #endif 3131 3132 static int rk3399_dmc_probe(struct udevice *dev) 3133 { 3134 #ifdef CONFIG_TPL_BUILD 3135 if (rk3399_dmc_init(dev)) 3136 return 0; 3137 #else 3138 struct dram_info *priv = dev_get_priv(dev); 3139 3140 priv->pmugrf = syscon_get_first_range(ROCKCHIP_SYSCON_PMUGRF); 3141 debug("%s: pmugrf=%p\n", __func__, priv->pmugrf); 3142 priv->info.base = CONFIG_SYS_SDRAM_BASE; 3143 priv->info.size = 3144 rockchip_sdram_size((phys_addr_t)&priv->pmugrf->os_reg2); 3145 #ifdef CONFIG_SPL_BUILD 3146 struct ddr_param ddr_parem; 3147 3148 ddr_parem.count = 1; 3149 ddr_parem.para[0] = priv->info.base; 3150 ddr_parem.para[1] = priv->info.size; 3151 rockchip_setup_ddr_param(&ddr_parem); 3152 #endif 3153 3154 #endif 3155 return 0; 3156 } 3157 3158 static int rk3399_dmc_get_info(struct udevice *dev, struct ram_info *info) 3159 { 3160 struct dram_info *priv = dev_get_priv(dev); 3161 3162 *info = priv->info; 3163 3164 return 0; 3165 } 3166 3167 static struct ram_ops rk3399_dmc_ops = { 3168 .get_info = rk3399_dmc_get_info, 3169 }; 3170 3171 static const struct udevice_id rk3399_dmc_ids[] = { 3172 { .compatible = "rockchip,rk3399-dmc" }, 3173 { } 3174 }; 3175 3176 U_BOOT_DRIVER(dmc_rk3399) = { 3177 .name = "rockchip_rk3399_dmc", 3178 .id = UCLASS_RAM, 3179 .of_match = rk3399_dmc_ids, 3180 .ops = &rk3399_dmc_ops, 3181 #ifdef CONFIG_TPL_BUILD 3182 .ofdata_to_platdata = rk3399_dmc_ofdata_to_platdata, 3183 #endif 3184 .probe = rk3399_dmc_probe, 3185 .priv_auto_alloc_size = sizeof(struct dram_info), 3186 #ifdef CONFIG_TPL_BUILD 3187 .platdata_auto_alloc_size = sizeof(struct rockchip_dmc_plat), 3188 #endif 3189 }; 3190