1 /* 2 * Copyright 2019-2023 NXP 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <bl31/interrupt_mgmt.h> 8 #include <common/runtime_svc.h> 9 #include <lib/mmio.h> 10 #include <lib/spinlock.h> 11 #include <plat/common/platform.h> 12 13 #include <dram.h> 14 #include <gpc.h> 15 16 #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10 17 #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11 18 19 struct dram_info dram_info; 20 21 /* lock used for DDR DVFS */ 22 spinlock_t dfs_lock; 23 24 #if defined(PLAT_imx8mq) 25 /* ocram used to dram timing */ 26 static uint8_t dram_timing_saved[13 * 1024] __aligned(8); 27 #endif 28 29 static volatile uint32_t wfe_done; 30 static volatile bool wait_ddrc_hwffc_done = true; 31 static unsigned int dev_fsp = 0x1; 32 33 static uint32_t fsp_init_reg[3][4] = { 34 { DDRC_INIT3(0), DDRC_INIT4(0), DDRC_INIT6(0), DDRC_INIT7(0) }, 35 { DDRC_FREQ1_INIT3(0), DDRC_FREQ1_INIT4(0), DDRC_FREQ1_INIT6(0), DDRC_FREQ1_INIT7(0) }, 36 { DDRC_FREQ2_INIT3(0), DDRC_FREQ2_INIT4(0), DDRC_FREQ2_INIT6(0), DDRC_FREQ2_INIT7(0) }, 37 }; 38 39 #if defined(PLAT_imx8mq) 40 static inline struct dram_cfg_param *get_cfg_ptr(void *ptr, 41 void *old_base, void *new_base) 42 { 43 uintptr_t offset = (uintptr_t)ptr & ~((uintptr_t)old_base); 44 45 return (struct dram_cfg_param *)(offset + new_base); 46 } 47 48 /* copy the dram timing info from DRAM to OCRAM */ 49 void imx8mq_dram_timing_copy(struct dram_timing_info *from) 50 { 51 struct dram_timing_info *info = (struct dram_timing_info *)dram_timing_saved; 52 53 /* copy the whole 13KB content used for dram timing info */ 54 memcpy(dram_timing_saved, from, sizeof(dram_timing_saved)); 55 56 /* correct the header after copied into ocram */ 57 info->ddrc_cfg = get_cfg_ptr(info->ddrc_cfg, from, dram_timing_saved); 58 info->ddrphy_cfg = get_cfg_ptr(info->ddrphy_cfg, from, dram_timing_saved); 59 info->ddrphy_trained_csr = get_cfg_ptr(info->ddrphy_trained_csr, from, dram_timing_saved); 60 info->ddrphy_pie = get_cfg_ptr(info->ddrphy_pie, from, dram_timing_saved); 61 } 62 #endif 63 64 #if defined(PLAT_imx8mp) 65 static uint32_t lpddr4_mr_read(unsigned int mr_rank, unsigned int mr_addr) 66 { 67 unsigned int tmp, drate_byte; 68 69 tmp = mmio_read_32(DRC_PERF_MON_MRR0_DAT(0)); 70 mmio_write_32(DRC_PERF_MON_MRR0_DAT(0), tmp | 0x1); 71 do { 72 tmp = mmio_read_32(DDRC_MRSTAT(0)); 73 } while (tmp & 0x1); 74 75 mmio_write_32(DDRC_MRCTRL0(0), (mr_rank << 4) | 0x1); 76 mmio_write_32(DDRC_MRCTRL1(0), (mr_addr << 8)); 77 mmio_write_32(DDRC_MRCTRL0(0), (mr_rank << 4) | BIT(31) | 0x1); 78 79 /* Workaround for SNPS STAR 9001549457 */ 80 do { 81 tmp = mmio_read_32(DDRC_MRSTAT(0)); 82 } while (tmp & 0x1); 83 84 do { 85 tmp = mmio_read_32(DRC_PERF_MON_MRR0_DAT(0)); 86 } while (!(tmp & 0x8)); 87 tmp = mmio_read_32(DRC_PERF_MON_MRR1_DAT(0)); 88 89 drate_byte = (mmio_read_32(DDRC_DERATEEN(0)) >> 4) & 0xff; 90 tmp = (tmp >> (drate_byte * 8)) & 0xff; 91 mmio_write_32(DRC_PERF_MON_MRR0_DAT(0), 0x4); 92 93 return tmp; 94 } 95 #endif 96 97 static void get_mr_values(uint32_t (*mr_value)[8]) 98 { 99 uint32_t init_val; 100 unsigned int i, fsp_index; 101 102 for (fsp_index = 0U; fsp_index < 3U; fsp_index++) { 103 for (i = 0U; i < 4U; i++) { 104 init_val = mmio_read_32(fsp_init_reg[fsp_index][i]); 105 mr_value[fsp_index][2*i] = init_val >> 16; 106 mr_value[fsp_index][2*i + 1] = init_val & 0xFFFF; 107 } 108 109 #if defined(PLAT_imx8mp) 110 if (dram_info.dram_type == DDRC_LPDDR4) { 111 mr_value[fsp_index][5] = lpddr4_mr_read(1, MR12); /* read MR12 from DRAM */ 112 mr_value[fsp_index][7] = lpddr4_mr_read(1, MR14); /* read MR14 from DRAM */ 113 } 114 #endif 115 } 116 } 117 118 static void save_rank_setting(void) 119 { 120 uint32_t i, offset; 121 uint32_t pstate_num = dram_info.num_fsp; 122 123 /* only support maximum 3 setpoints */ 124 pstate_num = (pstate_num > MAX_FSP_NUM) ? MAX_FSP_NUM : pstate_num; 125 126 for (i = 0U; i < pstate_num; i++) { 127 offset = i ? (i + 1) * 0x1000 : 0U; 128 dram_info.rank_setting[i][0] = mmio_read_32(DDRC_DRAMTMG2(0) + offset); 129 if (dram_info.dram_type != DDRC_LPDDR4) { 130 dram_info.rank_setting[i][1] = mmio_read_32(DDRC_DRAMTMG9(0) + offset); 131 } 132 #if !defined(PLAT_imx8mq) 133 dram_info.rank_setting[i][2] = mmio_read_32(DDRC_RANKCTL(0) + offset); 134 #endif 135 } 136 #if defined(PLAT_imx8mq) 137 dram_info.rank_setting[0][2] = mmio_read_32(DDRC_RANKCTL(0)); 138 #endif 139 } 140 /* Restore the ddrc configs */ 141 void dram_umctl2_init(struct dram_timing_info *timing) 142 { 143 struct dram_cfg_param *ddrc_cfg = timing->ddrc_cfg; 144 unsigned int i; 145 146 for (i = 0U; i < timing->ddrc_cfg_num; i++) { 147 mmio_write_32(ddrc_cfg->reg, ddrc_cfg->val); 148 ddrc_cfg++; 149 } 150 151 /* set the default fsp to P0 */ 152 mmio_write_32(DDRC_MSTR2(0), 0x0); 153 } 154 155 /* Restore the dram PHY config */ 156 void dram_phy_init(struct dram_timing_info *timing) 157 { 158 struct dram_cfg_param *cfg = timing->ddrphy_cfg; 159 unsigned int i; 160 161 /* Restore the PHY init config */ 162 cfg = timing->ddrphy_cfg; 163 for (i = 0U; i < timing->ddrphy_cfg_num; i++) { 164 dwc_ddrphy_apb_wr(cfg->reg, cfg->val); 165 cfg++; 166 } 167 168 /* Restore the DDR PHY CSRs */ 169 cfg = timing->ddrphy_trained_csr; 170 for (i = 0U; i < timing->ddrphy_trained_csr_num; i++) { 171 dwc_ddrphy_apb_wr(cfg->reg, cfg->val); 172 cfg++; 173 } 174 175 /* Load the PIE image */ 176 cfg = timing->ddrphy_pie; 177 for (i = 0U; i < timing->ddrphy_pie_num; i++) { 178 dwc_ddrphy_apb_wr(cfg->reg, cfg->val); 179 cfg++; 180 } 181 } 182 183 /* EL3 SGI-8 IPI handler for DDR Dynamic frequency scaling */ 184 static uint64_t waiting_dvfs(uint32_t id, uint32_t flags, 185 void *handle, void *cookie) 186 { 187 uint64_t mpidr = read_mpidr_el1(); 188 unsigned int cpu_id = MPIDR_AFFLVL0_VAL(mpidr); 189 uint32_t irq; 190 191 irq = plat_ic_acknowledge_interrupt(); 192 if (irq < 1022U) { 193 plat_ic_end_of_interrupt(irq); 194 } 195 196 /* set the WFE done status */ 197 spin_lock(&dfs_lock); 198 wfe_done |= (1 << cpu_id * 8); 199 dsb(); 200 spin_unlock(&dfs_lock); 201 202 while (1) { 203 /* ddr frequency change done */ 204 if (!wait_ddrc_hwffc_done) 205 break; 206 207 wfe(); 208 } 209 210 return 0; 211 } 212 213 void dram_info_init(unsigned long dram_timing_base) 214 { 215 uint32_t ddrc_mstr, current_fsp; 216 unsigned int idx = 0; 217 uint32_t flags = 0; 218 uint32_t rc; 219 unsigned int i; 220 221 /* Get the dram type & rank */ 222 ddrc_mstr = mmio_read_32(DDRC_MSTR(0)); 223 224 dram_info.dram_type = ddrc_mstr & DDR_TYPE_MASK; 225 dram_info.num_rank = ((ddrc_mstr >> 24) & ACTIVE_RANK_MASK) == 0x3 ? 226 DDRC_ACTIVE_TWO_RANK : DDRC_ACTIVE_ONE_RANK; 227 228 /* Get current fsp info */ 229 current_fsp = mmio_read_32(DDRC_DFIMISC(0)); 230 current_fsp = (current_fsp >> 8) & 0xf; 231 dram_info.boot_fsp = current_fsp; 232 dram_info.current_fsp = current_fsp; 233 234 #if defined(PLAT_imx8mq) 235 imx8mq_dram_timing_copy((struct dram_timing_info *)dram_timing_base); 236 dram_timing_base = (unsigned long) dram_timing_saved; 237 #endif 238 get_mr_values(dram_info.mr_table); 239 240 dram_info.timing_info = (struct dram_timing_info *)dram_timing_base; 241 242 /* get the num of supported fsp */ 243 for (i = 0U; i < 4U; ++i) { 244 if (!dram_info.timing_info->fsp_table[i]) { 245 break; 246 } 247 idx = i; 248 } 249 250 /* only support maximum 3 setpoints */ 251 dram_info.num_fsp = (i > MAX_FSP_NUM) ? MAX_FSP_NUM : i; 252 253 /* no valid fsp table, return directly */ 254 if (i == 0U) { 255 return; 256 } 257 258 /* save the DRAMTMG2/9 for rank to rank workaround */ 259 save_rank_setting(); 260 261 /* check if has bypass mode support */ 262 if (dram_info.timing_info->fsp_table[idx] < 666) { 263 dram_info.bypass_mode = true; 264 } else { 265 dram_info.bypass_mode = false; 266 } 267 268 /* Register the EL3 handler for DDR DVFS */ 269 set_interrupt_rm_flag(flags, NON_SECURE); 270 rc = register_interrupt_type_handler(INTR_TYPE_EL3, waiting_dvfs, flags); 271 if (rc != 0) { 272 panic(); 273 } 274 275 if (dram_info.dram_type == DDRC_LPDDR4 && current_fsp != 0x0) { 276 /* flush the L1/L2 cache */ 277 dcsw_op_all(DCCSW); 278 lpddr4_swffc(&dram_info, dev_fsp, 0x0); 279 dev_fsp = (~dev_fsp) & 0x1; 280 } else if (current_fsp != 0x0) { 281 /* flush the L1/L2 cache */ 282 dcsw_op_all(DCCSW); 283 ddr4_swffc(&dram_info, 0x0); 284 } 285 } 286 287 /* 288 * For each freq return the following info: 289 * 290 * r1: data rate 291 * r2: 1 + dram_core parent 292 * r3: 1 + dram_alt parent index 293 * r4: 1 + dram_apb parent index 294 * 295 * The parent indices can be used by an OS who manages source clocks to enabled 296 * them ahead of the switch. 297 * 298 * A parent value of "0" means "don't care". 299 * 300 * Current implementation of freq switch is hardcoded in 301 * plat/imx/common/imx8m/clock.c but in theory this can be enhanced to support 302 * a wide variety of rates. 303 */ 304 int dram_dvfs_get_freq_info(void *handle, u_register_t index) 305 { 306 switch (index) { 307 case 0: 308 SMC_RET4(handle, dram_info.timing_info->fsp_table[0], 309 1, 0, 5); 310 case 1: 311 if (!dram_info.bypass_mode) { 312 SMC_RET4(handle, dram_info.timing_info->fsp_table[1], 313 1, 0, 0); 314 } 315 SMC_RET4(handle, dram_info.timing_info->fsp_table[1], 316 2, 2, 4); 317 case 2: 318 if (!dram_info.bypass_mode) { 319 SMC_RET4(handle, dram_info.timing_info->fsp_table[2], 320 1, 0, 0); 321 } 322 SMC_RET4(handle, dram_info.timing_info->fsp_table[2], 323 2, 3, 3); 324 case 3: 325 SMC_RET4(handle, dram_info.timing_info->fsp_table[3], 326 1, 0, 0); 327 default: 328 SMC_RET1(handle, -3); 329 } 330 } 331 332 int dram_dvfs_handler(uint32_t smc_fid, void *handle, 333 u_register_t x1, u_register_t x2, u_register_t x3) 334 { 335 uint64_t mpidr = read_mpidr_el1(); 336 unsigned int cpu_id = MPIDR_AFFLVL0_VAL(mpidr); 337 unsigned int fsp_index = x1; 338 uint32_t online_cores = x2; 339 340 if (x1 == IMX_SIP_DDR_DVFS_GET_FREQ_COUNT) { 341 SMC_RET1(handle, dram_info.num_fsp); 342 } else if (x1 == IMX_SIP_DDR_DVFS_GET_FREQ_INFO) { 343 return dram_dvfs_get_freq_info(handle, x2); 344 } else if (x1 < 3U) { 345 wait_ddrc_hwffc_done = true; 346 dsb(); 347 348 /* trigger the SGI IPI to info other cores */ 349 for (int i = 0; i < PLATFORM_CORE_COUNT; i++) { 350 if (cpu_id != i && (online_cores & (0x1 << (i * 8)))) { 351 plat_ic_raise_el3_sgi(0x8, i); 352 } 353 } 354 #if defined(PLAT_imx8mq) 355 for (unsigned int i = 0; i < PLATFORM_CORE_COUNT; i++) { 356 if (i != cpu_id && online_cores & (1 << (i * 8))) { 357 imx_gpc_core_wake(1 << i); 358 } 359 } 360 #endif 361 /* make sure all the core in WFE */ 362 online_cores &= ~(0x1 << (cpu_id * 8)); 363 while (1) { 364 if (online_cores == wfe_done) { 365 break; 366 } 367 } 368 369 /* flush the L1/L2 cache */ 370 dcsw_op_all(DCCSW); 371 372 if (dram_info.dram_type == DDRC_LPDDR4) { 373 lpddr4_swffc(&dram_info, dev_fsp, fsp_index); 374 dev_fsp = (~dev_fsp) & 0x1; 375 } else { 376 ddr4_swffc(&dram_info, fsp_index); 377 } 378 379 dram_info.current_fsp = fsp_index; 380 wait_ddrc_hwffc_done = false; 381 wfe_done = 0; 382 dsb(); 383 sev(); 384 isb(); 385 } 386 387 SMC_RET1(handle, 0); 388 } 389