1 /*
2 * Copyright (c) 2020-2026, Renesas Electronics Corporation. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 /* include */
8 #include <common/debug.h>
9 #include <drivers/delay_timer.h>
10 #include <drivers/renesas/rza/cpg/cpg.h>
11 #include <drivers/renesas/rza/ddr/ddr_internal.h>
12
13 #define CEIL(a, div) (((a) + ((div) - 1)) / (div))
14 #define MAX_BYTE_LANES 2U
15 #define MAX_BEST_VREF_SAVED 30U
16 #define VREF_SETP 1U
17
18 /* prototypes */
19 void ddr_setup(void);
20 static void disable_phy_clk(void);
21 static void program_mc1(uint8_t *lp_auto_entry_en);
22 static void program_phy1(uint32_t sl_lanes, uint32_t byte_lanes);
23 static void exec_trainingWRLVL(uint32_t sl_lanes);
24 static void exec_trainingVREF(uint32_t sl_lanes, uint32_t byte_lanes);
25 static void setup_vref_training_registers(uint8_t vref_value, uint8_t cs,
26 uint8_t turn_on_off_vref_training);
27 static void write_mr(uint8_t cs, uint8_t mrw_sel, uint16_t mrw_data);
28 static void exec_trainingBITLVL(uint32_t sl_lanes);
29 static void opt_delay(uint32_t sl_lanes, uint32_t byte_lanes);
30 static void exec_trainingSL(uint32_t sl_lanes);
31 static void program_phy2(void);
32 static void program_mc2(void);
33
34 /* main */
ddr_setup(void)35 void ddr_setup(void)
36 {
37 uint32_t sl_lanes, byte_lanes;
38 uint8_t runBITLVL, runSL, runVREF;
39 uint8_t lp_auto_entry_en = 0;
40 uint32_t tmp;
41 int i;
42
43 NOTICE("Setup DDR (Rev. %s)\n", ddr_an_version);
44 /* Step2 - Step11 */
45 cpg_active_ddr(disable_phy_clk);
46
47 /* Step12 */
48 program_mc1(&lp_auto_entry_en);
49
50 /* Step13 */
51 tmp = read_mc_reg(DDRMC_R019);
52 sl_lanes = ((tmp & 0x1) == 0) ? 3 : 1;
53 byte_lanes = ((tmp & 0x1) == 0) ? 2 : 1;
54 tmp = read_mc_reg(DDRMC_R039);
55 runBITLVL = (tmp >> 20) & 0x1;
56 runSL = (tmp >> 21) & 0x1;
57 runVREF = (tmp >> 25) & 0x1;
58
59 /* Step14 */
60 program_phy1(sl_lanes, byte_lanes);
61
62 /* Step15 */
63 while ((read_phy_reg(DDRPHY_R42) & 0x00000003) != sl_lanes)
64 ;
65
66 /* Step16 */
67 ddr_ctrl_reten_en_n(false);
68 rmw_mc_reg(DDRMC_R007, 0xFFFFFEFF, 0x00000000);
69 rmw_mc_reg(DDRMC_R001, 0xFEFFFFFF, 0x01000000);
70 rmw_mc_reg(DDRMC_R000, 0xFFFFFFFE, 0x00000001);
71 while ((read_mc_reg(DDRMC_R021) & 0x02000000) != 0x02000000)
72 ;
73 rmw_phy_reg(DDRPHY_R74, 0xFFF7FFFF, 0x00080000);
74 rmw_mc_reg(DDRMC_R029, 0xFF0000FF, 64 << 8);
75 rmw_mc_reg(DDRMC_R027, 0xE00000FF, 111 << 8);
76 rmw_mc_reg(DDRMC_R020, 0xFFFFFEFF, 0x00000100);
77 udelay(1);
78 rmw_phy_reg(DDRPHY_R74, 0xFFF7FFFF, 0x00000000);
79
80 /* Step17 */
81 cpg_reset_ddr_mc();
82 ddr_ctrl_reten_en_n(true);
83
84 /* Step18-19 */
85 program_mc1(&lp_auto_entry_en);
86
87 /* Step20 */
88 for (i = 0; i < ARRAY_SIZE(swizzle_mc_tbl); i++) {
89 write_mc_reg(swizzle_mc_tbl[i][0], swizzle_mc_tbl[i][1]);
90 }
91 for (i = 0; i < ARRAY_SIZE(swizzle_phy_tbl); i++) {
92 write_phy_reg(swizzle_phy_tbl[i][0], swizzle_phy_tbl[i][1]);
93 }
94
95 /* Step21 */
96 rmw_mc_reg(DDRMC_R000, 0xFFFFFFFE, 0x00000001);
97
98 /* Step22 */
99 while ((read_mc_reg(DDRMC_R021) & 0x02000000) != 0x02000000)
100 ;
101
102 /* Step23 */
103 rmw_mc_reg(DDRMC_R023, 0xFDFFFFFF, 0x02000000);
104
105 /* Step24 */
106 exec_trainingWRLVL(sl_lanes);
107
108 /* Step25 */
109 if (runVREF == 1)
110 exec_trainingVREF(sl_lanes, byte_lanes);
111
112 /* Step26 */
113 if (runBITLVL == 1)
114 exec_trainingBITLVL(sl_lanes);
115
116 /* Step27 */
117 opt_delay(sl_lanes, byte_lanes);
118
119 /* Step28 */
120 if (runSL == 1)
121 exec_trainingSL(sl_lanes);
122
123 /* Step29 */
124 program_phy2();
125
126 /* Step30 */
127 program_mc2();
128
129 /* Step31 is skipped because ECC is unused. */
130
131 /* Step32 */
132 rmw_mc_reg(DDRMC_R006, 0xFFFFFFF0, lp_auto_entry_en & 0xF);
133 }
134
disable_phy_clk(void)135 static void disable_phy_clk(void)
136 {
137 /* Initialization Step9 */
138 write_phy_reg(DDRPHY_R77, 0x00000200);
139 write_phy_reg(DDRPHY_R78, 0x00010001);
140 }
141
program_mc1(uint8_t * lp_auto_entry_en)142 static void program_mc1(uint8_t *lp_auto_entry_en)
143 {
144 int i;
145
146 /* Step1 */
147 for (i = 0; i < ARRAY_SIZE(mc_init_tbl); i++) {
148 if (mc_init_tbl[i][0] == DDRMC_R006) {
149 *lp_auto_entry_en = mc_init_tbl[i][1] & 0xF;
150 write_mc_reg(DDRMC_R006,
151 mc_init_tbl[i][1] & 0xFFFFFFF0);
152 } else {
153 write_mc_reg(mc_init_tbl[i][0], mc_init_tbl[i][1]);
154 }
155 }
156
157 /* Step2 */
158 rmw_mc_reg(DDRMC_R025, 0xFCFFFFFF, mc_odt_pins_tbl[0] << 24);
159 rmw_mc_reg(DDRMC_R026, 0xFFFFFCFF, mc_odt_pins_tbl[1] << 8);
160 rmw_mc_reg(DDRMC_R025, 0xFFFCFFFF, mc_odt_pins_tbl[2] << 16);
161 rmw_mc_reg(DDRMC_R026, 0xFFFFFFFC, mc_odt_pins_tbl[3] << 0);
162
163 /* Step3 */
164 rmw_mc_reg(DDRMC_R009, ~(mc_mr1_tbl[0]), mc_mr1_tbl[1]);
165 rmw_mc_reg(DDRMC_R011, ~(mc_mr1_tbl[0]), mc_mr1_tbl[1]);
166
167 /* Step4 */
168 rmw_mc_reg(DDRMC_R010, ~(mc_mr2_tbl[0]), mc_mr2_tbl[1]);
169 rmw_mc_reg(DDRMC_R012, ~(mc_mr2_tbl[0]), mc_mr2_tbl[1]);
170
171 /* Step5 */
172 rmw_mc_reg(DDRMC_R015, ~(mc_mr5_tbl[0]), mc_mr5_tbl[1]);
173 rmw_mc_reg(DDRMC_R016, ~(mc_mr5_tbl[0]), mc_mr5_tbl[1]);
174
175 /* Step6 */
176 rmw_mc_reg(DDRMC_R017, ~(mc_mr6_tbl[0]), mc_mr6_tbl[1]);
177 rmw_mc_reg(DDRMC_R018, ~(mc_mr6_tbl[0]), mc_mr6_tbl[1]);
178
179 /* Step7 */
180 for (i = 0; i < ARRAY_SIZE(mc_phy_settings_tbl); i++) {
181 write_mc_reg(mc_phy_settings_tbl[i][0],
182 mc_phy_settings_tbl[i][1]);
183 }
184
185 /* Step8 is skipped because ECC is unused. */
186 }
187
program_phy1(uint32_t sl_lanes,uint32_t byte_lanes)188 static void program_phy1(uint32_t sl_lanes, uint32_t byte_lanes)
189 {
190 uint16_t dram_clk_period;
191 uint8_t dram;
192 uint8_t odt_wr_map_cs0, odt_rd_map_cs0;
193 uint8_t CL, CWL, AL, PL, RL, WL;
194 uint32_t mr1, mr1_wl, mr1_wl_mask;
195 uint32_t mr2, mr2_wl, mr2_wl_mask;
196 uint8_t clk_drive, dq_dqs_drive, dq_dqs_term, vref_value, vref_ca_value;
197 #if RZA3M
198 uint8_t adrctrl_drive;
199 #endif
200 uint8_t read_lat, trim_lat;
201 uint32_t tmp;
202 int i;
203
204 /* Step1 */
205 tmp = read_mc_reg(DDRMC_R039);
206 dram_clk_period = tmp & 0xFFFF;
207 dram = (tmp >> 16) & 0xF;
208
209 tmp = read_mc_reg(DDRMC_R025);
210 odt_wr_map_cs0 = (tmp >> 24) & 0x3;
211 odt_rd_map_cs0 = (tmp >> 16) & 0x3;
212
213 /* Step2 */
214 tmp = read_mc_reg(DDRMC_R002);
215 CL = (tmp >> 17) & 0x1F;
216 CWL = (tmp >> 24) & 0x1F;
217
218 tmp = read_mc_reg(DDRMC_R003);
219 AL = tmp & 0x1F;
220 PL = (tmp >> 8) & 0xF;
221
222 RL = CL + AL + PL;
223 WL = CWL + AL + PL;
224
225 /* Step3 */
226 mr1 = read_mc_reg(DDRMC_R009) & 0xFFFF;
227 mr2 = read_mc_reg(DDRMC_R010) & 0xFFFF;
228 if (dram == 2) {
229 /* DDR4 */
230 mr1_wl_mask = (0x7 << 8) | (0x1 << 7); /* 0x78 */
231 mr2_wl_mask = 0x7 << 9; /* 0xe0 */
232 switch ((mr2 & mr2_wl_mask) >> 9) {
233 case 0:
234 mr1_wl = 0x0;
235 mr1_wl_mask = 0x0000;
236 break;
237 case 1:
238 mr1_wl = 0x2 << 8;
239 break;
240 case 2:
241 mr1_wl = 0x4 << 8;
242 break;
243 case 4:
244 mr1_wl = 0x6 << 8;
245 break;
246 default:
247 panic();
248 }
249 } else {
250 /* DDR3L, DDR3 */
251 mr1_wl_mask = (0x1 << 9) | (0x1 << 7) | (0x1 << 6) | (0x1 << 2);
252 mr2_wl_mask = 0x3 << 9;
253 switch ((mr2 & mr2_wl_mask) >> 9) {
254 case 0:
255 mr1_wl = 0x0;
256 mr1_wl_mask = 0x0000;
257 break;
258 case 1:
259 mr1_wl = (0x0 << 9) | (0x0 << 6) | (0x1 << 2);
260 break;
261 case 2:
262 mr1_wl = (0x0 << 9) | (0x1 << 6) | (0x0 << 2);
263 break;
264 default:
265 panic();
266 }
267 }
268 mr1_wl |= (mr1 & (0xFFFF ^ mr1_wl_mask)) | (0x1 << 7);
269 mr2_wl = (mr2 & (0xFFFF ^ mr2_wl_mask)) | (0x0 << 9);
270
271 /* Step4 */
272 tmp = read_mc_reg(DDRMC_R040);
273 clk_drive = tmp & 0xF;
274 dq_dqs_drive = (tmp >> 4) & 0xF;
275 dq_dqs_term = (tmp >> 8) & 0xF;
276 #if RZA3M
277 adrctrl_drive = (tmp >> 12) & 0xF;
278 #endif
279 vref_value = (tmp >> 16) & 0xFF;
280 vref_ca_value = (tmp >> 24) & 0xFF;
281 read_lat = (dram == 2 ? 12 : 10) + (CEIL(RL, 2) * 2) - CEIL(WL, 2) + 28;
282 trim_lat = (dram == 2 ? 11 : 9) + CEIL(RL, 2) - CEIL(WL, 2) + 29;
283
284 /* Step5 */
285 tmp = ((WL == 5 ? 0x1 : 0x0) << 16) | 0x00100000;
286 write_phy_reg(DDRPHY_R77, tmp);
287
288 /* Step6 */
289 write_phy_reg(DDRPHY_R05, 0x00000006);
290
291 /* Step7 */
292 if (dram == 2) {
293 /* DDR4 */
294 write_phy_reg(DDRPHY_R65, 0x00000009);
295 }
296
297 /* Step8 */
298 write_phy_reg(DDRPHY_R67, (dram == 2 ? 1 : 0) << 27);
299 write_phy_reg(DDRPHY_R47, (dram == 2 ? 1 : 0) << 24);
300
301 /* Step9 */
302 tmp = ((dram == 0 ? 0 : 1) << 15) |
303 ((dram_clk_period < 1000 ? 1 : 0) << 8) | 0x10004000;
304 write_phy_reg(DDRPHY_R26, tmp);
305
306 /* Step10 */
307 #if RZA3M
308 write_phy_reg(DDRPHY_R13,
309 clk_drive | (clk_drive << 4) | (adrctrl_drive << 8) |
310 (adrctrl_drive << 12) | (dq_dqs_drive << 16) |
311 (dq_dqs_drive << 20));
312 #else
313 write_phy_reg(DDRPHY_R13, clk_drive | (clk_drive << 4) |
314 (clk_drive << 8) | (clk_drive << 12) |
315 (dq_dqs_drive << 16) |
316 (dq_dqs_drive << 20));
317 #endif
318
319 /* Step11 */
320 tmp = dq_dqs_term | ((dram == 2) ? 0 : (dq_dqs_term << 4));
321 write_phy_reg(DDRPHY_R14, tmp);
322
323 /* Step12 */
324 write_phy_reg(DDRPHY_R10, 0x00000000);
325
326 /* Step13 */
327 for (i = 0; i < byte_lanes; i++) {
328 write_phy_reg(DDRPHY_R29, 7 * i);
329 write_phy_reg(DDRPHY_R66, (vref_value << 4) | 0x00000004);
330 }
331
332 /* Step14 */
333 write_phy_reg(DDRPHY_R12, vref_ca_value);
334
335 /* Step15 */
336 write_phy_reg(DDRPHY_R61, 0x1A09002D);
337
338 /* Step16 */
339 write_phy_reg(DDRPHY_R41, 0x00000000);
340
341 /* Step17 */
342 write_phy_reg(DDRPHY_R74, 0x0000001A);
343
344 /* Step18 */
345 tmp = ((dram == 2 ? 0 : 1) << 2) | (CEIL(CL, 2) << 4) |
346 (CEIL(AL, 2) << 8) | (odt_rd_map_cs0 << 16) |
347 (odt_wr_map_cs0 << 24) | 0x00000001;
348 write_phy_reg(DDRPHY_R24, tmp);
349
350 /* Step19 */
351 tmp = (CEIL(CWL, 2) << 8) | ((((WL % 2) == 0) ? 0 : 1) << 24) |
352 0x80000001;
353 write_phy_reg(DDRPHY_R25, tmp);
354
355 /* Step20 */
356 write_phy_reg(DDRPHY_R45, sl_lanes ^ 0x3);
357
358 /* Step21 */
359 write_phy_reg(DDRPHY_R64, (trim_lat << 4) | (read_lat << 12));
360
361 /* Step22 */
362 tmp = ((WL % 2) == 0) & 0x1;
363 write_phy_reg(DDRPHY_R63, tmp);
364
365 /* Step23 */
366 write_phy_reg(DDRPHY_R72, 0x00000170);
367
368 /* Step24 */
369 write_phy_reg(DDRPHY_R38, mr2_wl | (mr2 << 16));
370
371 /* Step25 */
372 write_phy_reg(DDRPHY_R39, mr1 | (mr1_wl << 16));
373
374 /* Step26 */
375 write_phy_reg(DDRPHY_R27, 0xAC001000);
376
377 /* Step27 */
378 udelay(10);
379
380 /* Step28 is skipped because dll_mas_dly is unused. */
381
382 /* Step29 */
383 write_phy_reg(DDRPHY_R44, 0x00000000);
384
385 /* Step30 */
386 write_phy_reg(DDRPHY_R43, 0x00000000);
387
388 /* Step31 */
389 for (i = 0; i < byte_lanes; i++) {
390 write_phy_reg(DDRPHY_R29, 6 * i);
391 write_phy_reg(DDRPHY_R30, 9);
392 #if RZA3M
393 /* Changed from 10 to 14 according to APP_005 */
394 write_phy_reg(DDRPHY_R32, 14);
395 #else
396 write_phy_reg(DDRPHY_R32, 10);
397 #endif
398 }
399
400 /* Step32 */
401 write_phy_reg(DDRPHY_R28, 26 | 0x00000200);
402
403 /* Step33 */
404 write_phy_reg(DDRPHY_R29, 0);
405 #if RZA3M
406 write_phy_reg(DDRPHY_R57, 29 | 0x00000080);
407 #else
408 write_phy_reg(DDRPHY_R57, 26 | 0x00000080);
409 #endif
410
411 /* Step34 */
412 write_phy_reg(DDRPHY_R27, 26 | (0x10 << 8) | 0xAC000000);
413
414 /* Step35 */
415 write_phy_reg(DDRPHY_R21, 0x00035076);
416
417 /* Step36 */
418 write_phy_reg(DDRPHY_R07, 0x00000032);
419
420 /* lpddr4_combo_io_cal
421 * Step37-2 */
422 rmw_phy_reg(DDRPHY_R47, 0xFF800000, 0x00000000);
423
424 /* Step37-3 */
425 switch (dram) {
426 case 0:
427 tmp = 0x00003200;
428 break;
429 case 1:
430 tmp = 0x00005200;
431 break;
432 case 2:
433 tmp = 0x08009200;
434 break;
435 default:
436 tmp = 0x00000000;
437 break;
438 }
439 write_phy_reg(DDRPHY_R67, tmp);
440 write_phy_reg(DDRPHY_R46, 0x00000002);
441 while ((read_phy_reg(DDRPHY_R46) & 0x00000008) != 0x00000008)
442 ;
443
444 write_phy_reg(DDRPHY_R46, 0x00000000);
445 udelay(100);
446
447 /* Step37-4 */
448 switch (dram) {
449 case 0:
450 case 1:
451 tmp = 0x00041200;
452 break;
453 case 2:
454 tmp = 0x08101300;
455 break;
456 default:
457 tmp = 0x00000000;
458 break;
459 }
460 write_phy_reg(DDRPHY_R67, tmp);
461 write_phy_reg(DDRPHY_R46, 0x00000001);
462 while ((read_phy_reg(DDRPHY_R46) & 0x00000004) != 0x00000004)
463 ;
464
465 rmw_phy_reg(DDRPHY_R46, 0xFFFFFFEF, 0x00000010);
466 rmw_phy_reg(DDRPHY_R46, 0xFFFFFFEF, 0x00000000);
467 udelay(1);
468
469 /* Step38 */
470 rmw_phy_reg(DDRPHY_R27, 0xFBFFFFFF, 0x00000000);
471
472 /* Step39 */
473 rmw_phy_reg(DDRPHY_R78, 0xFFFFF0FE, (sl_lanes << 8));
474 }
475
exec_trainingWRLVL(uint32_t sl_lanes)476 static void exec_trainingWRLVL(uint32_t sl_lanes)
477 {
478 uint32_t tmp;
479
480 /* Step2 */
481 tmp = read_phy_reg(DDRPHY_R24);
482 write_phy_reg(DDRPHY_R24, tmp | 0x01000000);
483
484 /* Step3 */
485 write_phy_reg(DDRPHY_R37, sl_lanes);
486
487 /* Step4 */
488 write_phy_reg(DDRPHY_R31, 0x00010000);
489
490 /* Step5 */
491 write_phy_reg(DDRPHY_R18, 0x50200000);
492
493 /* Step6 */
494 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
495 ;
496
497 /* Step7 - Step8 */
498 if (((read_phy_reg(DDRPHY_R36) & sl_lanes) != sl_lanes) ||
499 ((read_phy_reg(DDRPHY_R37) & sl_lanes) != 0)) {
500 panic();
501 }
502
503 /* Step9 */
504 write_phy_reg(DDRPHY_R24, tmp);
505 }
506
exec_trainingVREF(uint32_t sl_lanes,uint32_t byte_lanes)507 static void exec_trainingVREF(uint32_t sl_lanes, uint32_t byte_lanes)
508 {
509 uint32_t vref_mid_level_code;
510 uint32_t vref_training_value;
511 uint32_t sweep_range;
512 uint16_t current_vref = 0;
513 uint32_t best_window_diff_so_far[MAX_BYTE_LANES];
514 uint32_t num_best_vref_matches[MAX_BYTE_LANES];
515 uint32_t all_best_vref_matches[MAX_BYTE_LANES][MAX_BEST_VREF_SAVED];
516 uint8_t window_0, window_1, window_diff;
517 uint32_t highest_best_vref_val, lowest_best_vref_val;
518 uint8_t orig_cs_config;
519 uint32_t tmp;
520 int i, j;
521
522 /* Step2 */
523 for (i = 0; i < byte_lanes; i++) {
524 write_phy_reg(DDRPHY_R29, i);
525 rmw_phy_reg(DDRPHY_R07, 0xFFFFFFCF, 0x00000010);
526 }
527 /* Step3 */
528 vref_mid_level_code = (read_mc_reg(DDRMC_R040) >> 16) & 0xFF;
529 sweep_range = read_mc_reg(DDRMC_R043) & 0xFF;
530
531 /* Step4 */
532 for (i = 0; i < byte_lanes; i++) {
533 best_window_diff_so_far[i] = 255;
534 num_best_vref_matches[i] = 0;
535 }
536
537 /* Step5 */
538 for (vref_training_value = 0;
539 vref_training_value < (sweep_range * 2) + 1;
540 vref_training_value += VREF_SETP) {
541 /* Step5.1 */
542 if (vref_training_value < sweep_range + 1) {
543 if (vref_mid_level_code < vref_training_value + 2) {
544 vref_training_value = sweep_range;
545 continue;
546 } else {
547 current_vref = vref_mid_level_code - vref_training_value;
548 }
549 } else {
550 if ((vref_mid_level_code + vref_training_value - sweep_range) > 126) {
551 break;
552 }
553
554 current_vref = vref_mid_level_code + vref_training_value - sweep_range;
555 }
556 for (i = 0; i < byte_lanes; i++) {
557 write_phy_reg(DDRPHY_R29, 7 * i);
558 write_phy_reg(DDRPHY_R66, (current_vref << 4) | 0x00000001);
559 }
560
561 /* Step5.2 */
562 write_phy_reg(DDRPHY_R18, 0x30800000);
563 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
564 ;
565
566 /* Step5.3 */
567 for (i = 0; i < byte_lanes; i++) {
568 if (((read_phy_reg(DDRPHY_R59) >> (14 + i)) & 0x1) == 0x0) {
569 INFO("PHY side VREF training passed on lane %0d, vref = %0d\n",
570 i, current_vref);
571 write_phy_reg(DDRPHY_R29, i * 6);
572 window_0 = read_phy_reg(DDRPHY_R69) & 0x3F;
573 window_1 = (read_phy_reg(DDRPHY_R69) >> 8) & 0x3F;
574 window_diff = (window_0 > window_1) ?
575 window_0 - window_1 :
576 window_1 - window_0;
577 INFO("window_0 = %0d, window_1 = %0d, window_diff = %0d\n",
578 window_0, window_1, window_diff);
579 if (window_diff < best_window_diff_so_far[i]) {
580 best_window_diff_so_far[i] = window_diff;
581 all_best_vref_matches[i][0] = current_vref;
582 num_best_vref_matches[i] = 1;
583 INFO("CURRENT BEST VREF PHY side :%d\n",
584 current_vref);
585 } else if ((window_diff == best_window_diff_so_far[i]) &&
586 (num_best_vref_matches[i] < MAX_BEST_VREF_SAVED)) {
587 all_best_vref_matches[i][num_best_vref_matches[i]] =
588 current_vref;
589 num_best_vref_matches[i] += 1;
590 }
591 } else {
592 INFO("PHY side VREF training failed lane %d, current_vref = %d\n",
593 i, current_vref);
594 }
595 }
596 /* Step5.4 */
597 }
598
599 /* Step6 */
600 for (i = 0; i < byte_lanes; i++) {
601 highest_best_vref_val = 0x0;
602 lowest_best_vref_val = 0x7F;
603 for (j = 0; j < num_best_vref_matches[i]; j++) {
604 highest_best_vref_val = MAX(all_best_vref_matches[i][j],
605 highest_best_vref_val);
606 lowest_best_vref_val = MIN(all_best_vref_matches[i][j],
607 lowest_best_vref_val);
608 }
609 current_vref = (highest_best_vref_val + lowest_best_vref_val) >> 1;
610 write_phy_reg(DDRPHY_R29, 7 * i);
611 write_phy_reg(DDRPHY_R66, current_vref << 4);
612 }
613
614 /* Step7 */
615 write_phy_reg(DDRPHY_R19, 0xFF00FF00);
616 write_phy_reg(DDRPHY_R20, 0xFF00FF00);
617
618 /* Step8 */
619 write_phy_reg(DDRPHY_R18, 0x30800000);
620 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
621 ;
622
623 /* Step9 */
624 tmp = (read_phy_reg(DDRPHY_R59) >> 14) & sl_lanes;
625 if ((tmp ^ sl_lanes) != sl_lanes) {
626 panic();
627 }
628
629 /* Step10 */
630 rmw_phy_reg(DDRPHY_R54, 0xFFFFFF7F, 0x00000080);
631
632 /* Step11 */
633 vref_mid_level_code = (read_mc_reg(DDRMC_R043) >> 8) & 0xFF;
634 sweep_range = (read_mc_reg(DDRMC_R043) >> 16) & 0xFF;
635
636 /* Step12 */
637 orig_cs_config = read_phy_reg(DDRPHY_R25) & 0x3;
638
639 /* Step13 */
640 setup_vref_training_registers(vref_mid_level_code, sl_lanes, 1);
641
642 /* Step14 */
643 rmw_phy_reg(DDRPHY_R66, 0xFFFFFFFE, 0x00000001);
644
645 /* Step15 */
646 for (i = 0; i < byte_lanes; i++) {
647 best_window_diff_so_far[i] = 255;
648 num_best_vref_matches[i] = 0;
649 }
650
651 /* Step16 */
652 for (vref_training_value = 0;
653 vref_training_value < (sweep_range * 2) + 1;
654 vref_training_value += VREF_SETP) {
655 /* Step16.1 */
656 if (vref_training_value < (sweep_range + 1)) {
657 if (vref_training_value > vref_mid_level_code) {
658 vref_training_value = sweep_range;
659 continue;
660 } else {
661 current_vref = vref_mid_level_code - vref_training_value;
662 }
663 } else {
664 if ((vref_mid_level_code + vref_training_value - sweep_range) <= 73) {
665 current_vref = vref_mid_level_code + vref_training_value -
666 sweep_range;
667 } else {
668 break;
669 }
670 }
671 setup_vref_training_registers(current_vref, orig_cs_config, 0);
672
673 /* Step16.2 */
674 write_phy_reg(DDRPHY_R18, 0x30500000);
675 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
676 ;
677
678 /* Step16.3 */
679 tmp = (read_phy_reg(DDRPHY_R64) >> 20) & sl_lanes;
680 for (i = 0; i < byte_lanes; i++) {
681 if ((tmp ^ sl_lanes) == sl_lanes) {
682 INFO("VREF training passed, vref = %d\n", current_vref);
683 write_phy_reg(DDRPHY_R29, i * 6);
684 window_0 = read_phy_reg(DDRPHY_R69) & 0x3F;
685 window_1 = (read_phy_reg(DDRPHY_R69) >> 8) & 0x3F;
686 window_diff = (window_0 > window_1) ?
687 window_0 - window_1 :
688 window_1 - window_0;
689 INFO("window_0 = %0d, window_1 = %0d, window_diff = %0d\n",
690 window_0, window_1, window_diff);
691 if (window_diff < best_window_diff_so_far[i]) {
692 best_window_diff_so_far[i] = window_diff;
693 all_best_vref_matches[i][0] = current_vref;
694 num_best_vref_matches[i] = 1;
695 INFO("CURRENT BEST VREF DRAM side :%d\n",
696 current_vref);
697 } else if ((window_diff == best_window_diff_so_far[i]) &&
698 (num_best_vref_matches[i] < MAX_BEST_VREF_SAVED)) {
699 all_best_vref_matches[i][num_best_vref_matches[i]] =
700 current_vref;
701 num_best_vref_matches[i] += 1;
702 }
703 } else {
704 INFO("VREF training failed, vref = %d\n", current_vref);
705 }
706 }
707 /* Step16.4 */
708 }
709
710 /* Step17 */
711 highest_best_vref_val = 0x0;
712 lowest_best_vref_val = 0x7F;
713 for (i = 0; i < byte_lanes; i++) {
714 for (j = 0; j < num_best_vref_matches[i]; j++) {
715 highest_best_vref_val = MAX(all_best_vref_matches[i][j],
716 highest_best_vref_val);
717 lowest_best_vref_val = MIN(all_best_vref_matches[i][j],
718 lowest_best_vref_val);
719 }
720 }
721 current_vref = (highest_best_vref_val + lowest_best_vref_val) >> 1;
722
723 /* Step18 */
724 setup_vref_training_registers(current_vref, sl_lanes, 0);
725
726 /* Step19 */
727 rmw_mc_reg(DDRMC_R044, 0xFFFFFF00, current_vref);
728
729 /* Step20 */
730 rmw_phy_reg(DDRPHY_R66, 0xFFFFFFFE, 0x00000000);
731
732 /* Step21 */
733 setup_vref_training_registers(current_vref, sl_lanes, 2);
734
735 /* Step22 */
736 rmw_phy_reg(DDRPHY_R54, 0xFFFFFF7F, 0x00000000);
737
738 /* Step23 */
739 for (i = 0; i < byte_lanes; i++) {
740 write_phy_reg(DDRPHY_R29, i);
741 rmw_phy_reg(DDRPHY_R07, 0xFFFFFFCF, 0x00000030);
742 }
743 }
744
setup_vref_training_registers(uint8_t vref_value,uint8_t cs,uint8_t turn_on_off_vref_training)745 static void setup_vref_training_registers(uint8_t vref_value, uint8_t cs,
746 uint8_t turn_on_off_vref_training)
747 {
748 uint8_t vref_op_code;
749 uint16_t mr;
750
751 /* Step1 */
752 if (vref_value > 50) {
753 vref_op_code = vref_value - 23;
754 } else {
755 vref_op_code = vref_value | (1 << 6);
756 }
757
758 /* Step2 */
759 mr = read_mc_reg(DDRMC_R017) & 0xFF00;
760 write_mr(cs, 6,
761 mr | (((turn_on_off_vref_training == 2) ? 0 : 1) << 7) |
762 vref_op_code);
763
764 /* Step3 */
765 udelay(1);
766 }
767
write_mr(uint8_t cs,uint8_t mrw_sel,uint16_t mrw_data)768 static void write_mr(uint8_t cs, uint8_t mrw_sel, uint16_t mrw_data)
769 {
770 uint8_t mrw_cs;
771 uint8_t mrw_allcs;
772
773 /* Step1 */
774 mrw_cs = 0;
775 if (cs & 0x1) {
776 rmw_mc_reg(DDRMC_R013, 0xFFFF0000, mrw_data);
777 mrw_cs = 0;
778 }
779 if (cs & 0x2) {
780 rmw_mc_reg(DDRMC_R014, 0xFFFF0000, mrw_data);
781 mrw_cs = 1;
782 }
783 mrw_allcs = ((cs & 0x3) == 0x3) ? 1 : 0;
784
785 /* Step2 */
786 rmw_mc_reg(DDRMC_R008, 0xFC000000,
787 0x02800000 | (mrw_allcs << 24) | (mrw_cs << 8) | mrw_sel);
788
789 /* Step3 */
790 while ((read_mc_reg(DDRMC_R022) & (1 << 3)) != (1 << 3))
791 ;
792
793 /* Step4 */
794 rmw_mc_reg(DDRMC_R024, 0xFFFFFFF7, 0x00000008);
795 }
796
exec_trainingBITLVL(uint32_t sl_lanes)797 static void exec_trainingBITLVL(uint32_t sl_lanes)
798 {
799 uint32_t tmp;
800
801 /* Step2 */
802 write_phy_reg(DDRPHY_R62, 0x00000000);
803
804 /* Step3 */
805 write_phy_reg(DDRPHY_R19, 0xFF00FF00);
806 write_phy_reg(DDRPHY_R20, 0xFF00FF00);
807
808 /* Step4 */
809 write_phy_reg(DDRPHY_R18, 0x30A00000);
810
811 /* Step5 */
812 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
813 ;
814
815 /* Step6 */
816 tmp = (read_phy_reg(DDRPHY_R59) >> 14) & sl_lanes;
817 if ((tmp ^ sl_lanes) != sl_lanes) {
818 panic();
819 }
820
821 /* Step7 */
822 rmw_phy_reg(DDRPHY_R54, 0xFFFFFF7F, 0x00000080);
823
824 /* Step8 */
825 write_phy_reg(DDRPHY_R18, 0x30700000);
826
827 /* Step9 */
828 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
829 ;
830
831 /* Step10 */
832 tmp = (read_phy_reg(DDRPHY_R64) >> 20) & sl_lanes;
833 if ((tmp ^ sl_lanes) != sl_lanes) {
834 panic();
835 }
836
837 /* Step11 */
838 rmw_phy_reg(DDRPHY_R54, 0xFFFFFF7F, 0x00000000);
839
840 /* Step12 */
841 write_phy_reg(DDRPHY_R51, 0x00080000);
842
843 /* Step13 */
844 write_phy_reg(DDRPHY_R18, 0x11200000);
845
846 /* Step14 */
847 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
848 ;
849
850 /* Step15 */
851 write_phy_reg(DDRPHY_R51, 0x00000000);
852
853 /* Step16 */
854 write_phy_reg(DDRPHY_R18, 0x30A00000);
855
856 /* Step17 */
857 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
858 ;
859 }
860
opt_delay(uint32_t sl_lanes,uint32_t byte_lanes)861 static void opt_delay(uint32_t sl_lanes, uint32_t byte_lanes)
862 {
863 uint32_t tmp;
864 uint16_t dlls_trim_ca;
865 #if RZA3M
866 uint16_t dlls_trim_clk;
867 #endif
868 uint16_t dlls_trim_2[MAX_BYTE_LANES];
869 uint16_t op_dqs_trim[MAX_BYTE_LANES];
870 uint16_t min_WL;
871 uint16_t min_WD = 128;
872 int i, j;
873
874 /* Step2 */
875 rmw_phy_reg(DDRPHY_R27, 0xFBFFFFFF, 0x04000000);
876
877 /* Step3 */
878 rmw_mc_reg(DDRMC_R004, ~(0x7F << LP_CMD_OFFSET),
879 (0x00000011 << LP_CMD_OFFSET));
880 while (((read_mc_reg(DDRMC_R005) >> 24) & 0x7F) != 0x48)
881 ;
882
883 /* Step4 */
884 write_phy_reg(DDRPHY_R29, 0);
885 #if RZA3M
886 dlls_trim_clk = read_phy_reg(DDRPHY_R57) & 0x7F;
887 dlls_trim_ca = read_phy_reg(DDRPHY_R28) & 0x7F;
888 if (dlls_trim_clk <= dlls_trim_ca) {
889 panic();
890 }
891 #else
892 dlls_trim_ca = read_phy_reg(DDRPHY_R57) & 0x7F;
893 #endif
894 min_WL = dlls_trim_ca;
895
896 for (i = 0; i < byte_lanes; i++) {
897 write_phy_reg(DDRPHY_R29, 6 * i);
898 dlls_trim_2[i] = read_phy_reg(DDRPHY_R31) & 0x3F;
899 min_WL = MIN(min_WL, dlls_trim_2[i]);
900
901 write_phy_reg(DDRPHY_R29, (7 * i) | 0x00000900);
902 op_dqs_trim[i] = read_phy_reg(DDRPHY_R56) & 0x3F;
903 min_WD = MIN(min_WD, op_dqs_trim[i]);
904 for (j = 0; j < 9; j++) {
905 write_phy_reg(DDRPHY_R29, (i * 7) | (j << 8));
906 tmp = read_phy_reg(DDRPHY_R56) & 0x7F;
907 tmp = (tmp & 0x40) ? (op_dqs_trim[i] + (tmp & 0x3F)) :
908 (op_dqs_trim[i] - (tmp & 0x3F));
909 min_WD = MIN((uint32_t)min_WD, tmp);
910 }
911 }
912
913 /* Step5 */
914 #if RZA3M
915 tmp = (dlls_trim_clk - min_WL) & 0x7F;
916 #else
917 tmp = (dlls_trim_ca - min_WL) & 0x7F;
918 #endif
919 write_phy_reg(DDRPHY_R29, 0);
920 write_phy_reg(DDRPHY_R57, tmp | 0x00000080);
921 #if RZA3M
922 tmp = (dlls_trim_ca - min_WL) & 0x7F;
923 #endif
924 write_phy_reg(DDRPHY_R28, tmp | 0x00000200);
925 rmw_phy_reg(DDRPHY_R27, 0xFFFFFF80, tmp);
926
927 for (i = 0; i < byte_lanes; i++) {
928 tmp = (dlls_trim_2[i] - min_WL) & 0x3F;
929 write_phy_reg(DDRPHY_R29, 6 * i);
930 rmw_phy_reg(DDRPHY_R31, 0xFFFFFFC0, tmp);
931
932 write_phy_reg(DDRPHY_R29, (7 * i) | 0x00000900);
933 tmp = (op_dqs_trim[i] - min_WD) & 0x3F;
934 rmw_phy_reg(DDRPHY_R56, 0xFFFFFF80, tmp);
935 }
936
937 /* Step6 */
938 rmw_mc_reg(DDRMC_R004, ~(0x7F << LP_CMD_OFFSET),
939 (0x00000002 << LP_CMD_OFFSET));
940 while (((read_mc_reg(DDRMC_R005) >> 24) & 0x7F) != 0x40)
941 ;
942
943 /* Step6 */
944 rmw_phy_reg(DDRPHY_R27, 0xFBFFFFFF, 0x00000000);
945 while ((read_phy_reg(DDRPHY_R42) & 0x3) != sl_lanes)
946 ;
947 }
948
exec_trainingSL(uint32_t sl_lanes)949 static void exec_trainingSL(uint32_t sl_lanes)
950 {
951 /* Step2 */
952 write_phy_reg(DDRPHY_R62, 0x00000001);
953
954 /* Step3 */
955 write_phy_reg(DDRPHY_R34, 0x00000010);
956
957 /* Step4 */
958 write_phy_reg(DDRPHY_R19, 0x789B3DE0);
959 write_phy_reg(DDRPHY_R20, 0xF10E4A56);
960
961 /* Step5 */
962 write_phy_reg(DDRPHY_R18, 0x11200000);
963
964 /* Step6 */
965 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
966 ;
967
968 /* Step7 */
969 write_phy_reg(DDRPHY_R18, 0x11200000);
970
971 /* Step8 */
972 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
973 ;
974
975 /* Step9 */
976 write_phy_reg(DDRPHY_R18, 0x34200000);
977
978 /* Step10 */
979 while ((read_phy_reg(DDRPHY_R18) & 0x10000000) != 0x00000000)
980 ;
981
982 /* Step11 */
983 if ((read_phy_reg(DDRPHY_R18) & sl_lanes) != sl_lanes) {
984 panic();
985 }
986
987 /* Step12 */
988 write_phy_reg(DDRPHY_R62, 0x00000003);
989 }
990
program_phy2(void)991 static void program_phy2(void)
992 {
993 uint16_t dram_clk_period;
994 uint32_t tmp, b21, b22, b23;
995
996 /* Step1 */
997 tmp = read_mc_reg(DDRMC_R039);
998 dram_clk_period = tmp & 0xFFFF;
999 b21 = (tmp >> 21) & 0x1;
1000 b22 = (tmp >> 22) & 0x1;
1001 b23 = (tmp >> 23) & 0x1;
1002
1003 /* Step2 */
1004 tmp = 1000000000000UL / (2 * dram_clk_period * 256);
1005 rmw_phy_reg(DDRPHY_R64, 0xFFFFFFFE, b23);
1006 rmw_phy_reg(DDRPHY_R59, 0xFFFFFFFE, (b23 == 1 ? 0 : b22));
1007 write_phy_reg(DDRPHY_R55, (b21 << 24) | MIN(tmp, 0xFFFFFFU));
1008
1009 /* Step3 */
1010 tmp = 1000000000000UL / (dram_clk_period * 256);
1011 rmw_phy_reg(DDRPHY_R27, 0xFBFFFFFF, 0x04000000);
1012 rmw_phy_reg(DDRPHY_R27, 0xFC0000FF, MIN(tmp, 0x3FFFFU) << 8);
1013 rmw_phy_reg(DDRPHY_R27, 0xFBFFFFFF, 0x00000000);
1014 }
1015
program_mc2(void)1016 static void program_mc2(void)
1017 {
1018 uint8_t main_clk_dly;
1019 uint8_t tphy_rdlat;
1020 uint32_t tmp;
1021
1022 /* Step1 */
1023 main_clk_dly = (read_phy_reg(DDRPHY_R21) >> 4) & 0xF;
1024 tmp = (read_mc_reg(DDRMC_R028) >> 24) & 0x7F;
1025 tphy_rdlat = ((main_clk_dly + 1 + 1) * 2) + 2 + ((tmp == 1) ? 2 : 0);
1026
1027 /* Step2 */
1028 rmw_mc_reg(DDRMC_R027, 0xFFFFFF80, tphy_rdlat & 0x7F);
1029 }
1030